diff options
12 files changed, 527 insertions, 855 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java deleted file mode 100644 index cf010121c2a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Duration; -import java.time.Instant; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author freva - */ -public class FileHelper { - private static final Logger logger = Logger.getLogger(FileHelper.class.getSimpleName()); - - /** - * (Recursively) deletes files if they match all the criteria, also deletes empty directories. - * - * @param basePath Base path from where to start the search - * @param maxAge Delete files older (last modified date) than maxAge - * @param fileNameRegex Delete files where filename matches fileNameRegex - * @param recursive Delete files in sub-directories (with the same criteria) - */ - public static void deleteFiles(Path basePath, Duration maxAge, Optional<String> fileNameRegex, boolean recursive) throws IOException { - Pattern fileNamePattern = fileNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - if (recursive) { - deleteFiles(path, maxAge, fileNameRegex, true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } else if (isPatternMatchingFilename(fileNamePattern, path) && - isTimeSinceLastModifiedMoreThan(path, maxAge)) { - if (! Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes all files in target directory except the n most recent (by modified date) - * - * @param basePath Base path to delete from - * @param nMostRecentToKeep Number of most recent files to keep - */ - static void deleteFilesExceptNMostRecent(Path basePath, int nMostRecentToKeep) throws IOException { - if (nMostRecentToKeep < 1) { - throw new IllegalArgumentException("Number of files to keep must be a positive number"); - } - - List<Path> pathsInDeleteDir = listContentsOfDirectory(basePath).stream() - .filter(Files::isRegularFile) - .sorted(Comparator.comparing(FileHelper::getLastModifiedTime)) - .skip(nMostRecentToKeep) - .collect(Collectors.toList()); - - for (Path path : pathsInDeleteDir) { - if (!Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - - static void deleteFilesLargerThan(Path basePath, long sizeInBytes) throws IOException { - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - deleteFilesLargerThan(path, sizeInBytes); - } else { - if (Files.size(path) > sizeInBytes && !Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes directories and their contents if they match all the criteria - * - * @param basePath Base path to delete the directories from - * @param maxAge Delete directories older (last modified date) than maxAge - * @param dirNameRegex Delete directories where directory name matches dirNameRegex - */ - public static void deleteDirectories(Path basePath, Duration maxAge, Optional<String> dirNameRegex) throws IOException { - Pattern dirNamePattern = dirNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path) && isPatternMatchingFilename(dirNamePattern, path)) { - boolean mostRecentFileModifiedBeforeMaxAge = getMostRecentlyModifiedFileIn(path) - .map(mostRecentlyModified -> isTimeSinceLastModifiedMoreThan(mostRecentlyModified, maxAge)) - .orElse(true); - - if (mostRecentFileModifiedBeforeMaxAge) { - deleteFiles(path, Duration.ZERO, Optional.empty(), true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } - } - } - - /** - * Similar to rm -rf file: - * - It's not an error if file doesn't exist - * - If file is a directory, it and all content is removed - * - For symlinks: Only the symlink is removed, not what the symlink points to - */ - public static void recursiveDelete(Path basePath) throws IOException { - if (Files.isDirectory(basePath)) { - for (Path path : listContentsOfDirectory(basePath)) { - recursiveDelete(path); - } - } - - Files.deleteIfExists(basePath); - } - - public static void moveIfExists(Path from, Path to) throws IOException { - if (Files.exists(from)) { - Files.move(from, to); - } - } - - private static Optional<Path> getMostRecentlyModifiedFileIn(Path basePath) throws IOException { - return Files.walk(basePath).max(Comparator.comparing(FileHelper::getLastModifiedTime)); - } - - private static boolean isTimeSinceLastModifiedMoreThan(Path path, Duration duration) { - Instant nowMinusDuration = Instant.now().minus(duration); - Instant lastModified = getLastModifiedTime(path).toInstant(); - - // Return true also if they are equal for test stability - // (lastModified <= nowMinusDuration) is the same as !(lastModified > nowMinusDuration) - return !lastModified.isAfter(nowMinusDuration); - } - - private static boolean isPatternMatchingFilename(Pattern pattern, Path path) { - return pattern == null || pattern.matcher(path.getFileName().toString()).find(); - } - - /** - * @return list all files in a directory, returns empty list if directory does not exist - */ - public static List<Path> listContentsOfDirectory(Path basePath) { - try (Stream<Path> directoryStream = Files.list(basePath)) { - return directoryStream.collect(Collectors.toList()); - } catch (NoSuchFileException ignored) { - return Collections.emptyList(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to list contents of directory " + basePath.toAbsolutePath(), e); - } - } - - static FileTime getLastModifiedTime(Path path) { - try { - return Files.getLastModifiedTime(path, LinkOption.NOFOLLOW_LINKS); - } catch (IOException e) { - throw new UncheckedIOException("Failed to get last modified time of " + path.toAbsolutePath(), e); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 236415d1bcd..1affe890eee 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -1,23 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.collections.Pair; import com.yahoo.config.provision.NodeType; import com.yahoo.io.IOUtils; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; -import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; -import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileHelper; import com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig; @@ -25,13 +19,10 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; import java.io.IOException; import java.io.InputStreamReader; -import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.time.Clock; import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -39,7 +30,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -50,41 +40,18 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults; * @author freva */ public class StorageMaintainer { - private static final ContainerName NODE_ADMIN = new ContainerName("node-admin"); - private static final ObjectMapper objectMapper = new ObjectMapper(); - private final GaugeWrapper numberOfCoredumpsOnHost; - private final CounterWrapper numberOfNodeAdminMaintenanceFails; private final DockerOperations dockerOperations; private final ProcessExecuter processExecuter; private final Environment environment; - private final Optional<CoredumpHandler> coredumpHandler; - private final Clock clock; - - private final Map<ContainerName, MaintenanceThrottler> maintenanceThrottlerByContainerName = new ConcurrentHashMap<>(); - - public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter, - MetricReceiverWrapper metricReceiver, Environment environment, Clock clock) { - this(dockerOperations, processExecuter, metricReceiver, environment, null, clock); - } + private final CoredumpHandler coredumpHandler; public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter, - MetricReceiverWrapper metricReceiver, Environment environment, - CoredumpHandler coredumpHandler, Clock clock) { + Environment environment, CoredumpHandler coredumpHandler) { this.dockerOperations = dockerOperations; this.processExecuter = processExecuter; this.environment = environment; - this.coredumpHandler = Optional.ofNullable(coredumpHandler); - this.clock = clock; - - Dimensions dimensions = new Dimensions.Builder() - .add("role", SecretAgentCheckConfig.nodeTypeToRole(environment.getNodeType())) - .build(); - numberOfNodeAdminMaintenanceFails = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.maintenance.fails"); - numberOfCoredumpsOnHost = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.coredumps"); - - metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.running_on_host") - .add(environment.isRunningOnHost() ? 1 : 0); + this.coredumpHandler = coredumpHandler; } public void writeMetricsConfig(ContainerName containerName, NodeSpec node) { @@ -241,17 +208,7 @@ public class StorageMaintainer { * Deletes old log files for vespa, nginx, logstash, etc. */ public void removeOldFilesFromNode(ContainerName containerName) { - if (! getMaintenanceThrottlerFor(containerName).shouldRemoveOldFilesNow()) return; - - MaintainerExecutor maintainerExecutor = new MaintainerExecutor(); - addRemoveOldFilesCommand(maintainerExecutor, containerName); - - maintainerExecutor.execute(); - getMaintenanceThrottlerFor(containerName).updateNextRemoveOldFilesTime(); - } - - private void addRemoveOldFilesCommand(MaintainerExecutor maintainerExecutor, ContainerName containerName) { - Path[] pathsToClean = { + Path[] logPaths = { environment.pathInNodeUnderVespaHome("logs/elasticsearch2"), environment.pathInNodeUnderVespaHome("logs/logstash2"), environment.pathInNodeUnderVespaHome("logs/daemontools_y"), @@ -259,79 +216,42 @@ public class StorageMaintainer { environment.pathInNodeUnderVespaHome("logs/vespa") }; - for (Path pathToClean : pathsToClean) { + for (Path pathToClean : logPaths) { Path path = environment.pathInNodeAdminFromPathInNode(containerName, pathToClean); - if (Files.exists(path)) { - maintainerExecutor.addJob("delete-files") - .withArgument("basePath", path) - .withArgument("maxAgeSeconds", Duration.ofDays(3).getSeconds()) - .withArgument("fileNameRegex", ".*\\.log.+") - .withArgument("recursive", false); - } + FileHelper.streamFiles(path) + .filterFile(FileHelper.olderThan(Duration.ofDays(3)) + .and(FileHelper.nameMatches(Pattern.compile(".*\\.log.+")))) + .delete(); } Path qrsDir = environment.pathInNodeAdminFromPathInNode( containerName, environment.pathInNodeUnderVespaHome("logs/vespa/qrs")); - maintainerExecutor.addJob("delete-files") - .withArgument("basePath", qrsDir) - .withArgument("maxAgeSeconds", Duration.ofDays(3).getSeconds()) - .withArgument("recursive", false); + FileHelper.streamFiles(qrsDir) + .filterFile(FileHelper.olderThan(Duration.ofDays(3))) + .delete(); Path logArchiveDir = environment.pathInNodeAdminFromPathInNode( containerName, environment.pathInNodeUnderVespaHome("logs/vespa/logarchive")); - maintainerExecutor.addJob("delete-files") - .withArgument("basePath", logArchiveDir) - .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds()) - .withArgument("recursive", false); + FileHelper.streamFiles(logArchiveDir) + .filterFile(FileHelper.olderThan(Duration.ofDays(31))) + .delete(); Path fileDistrDir = environment.pathInNodeAdminFromPathInNode( containerName, environment.pathInNodeUnderVespaHome("var/db/vespa/filedistribution")); - maintainerExecutor.addJob("delete-files") - .withArgument("basePath", fileDistrDir) - .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds()) - .withArgument("recursive", true); + FileHelper.streamFiles(fileDistrDir) + .filterFile(FileHelper.olderThan(Duration.ofDays(31))) + .recursive(true) + .delete(); } /** * Checks if container has any new coredumps, reports and archives them if so */ public void handleCoreDumpsForContainer(ContainerName containerName, NodeSpec node) { - // Sample number of coredumps on the host - try (Stream<Path> files = Files.list(environment.pathInNodeAdminToDoneCoredumps())) { - numberOfCoredumpsOnHost.sample(files.count()); - } catch (IOException e) { - // Ignore for now - this is either test or a misconfiguration - } - - MaintainerExecutor maintainerExecutor = new MaintainerExecutor(); - addHandleCoredumpsCommand(maintainerExecutor, containerName, node); - maintainerExecutor.execute(); - } - - /** - * Will either schedule coredump execution in the given maintainerExecutor or run coredump handling - * directly if {@link #coredumpHandler} is set. - */ - private void addHandleCoredumpsCommand(MaintainerExecutor maintainerExecutor, ContainerName containerName, NodeSpec node) { final Path coredumpsPath = environment.pathInNodeAdminFromPathInNode( containerName, environment.pathInNodeUnderVespaHome("var/crash")); final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(node); - if (coredumpHandler.isPresent()) { - try { - coredumpHandler.get().processAll(coredumpsPath, nodeAttributes); - } catch (IOException e) { - throw new UncheckedIOException("Failed to process coredumps", e); - } - } else { - // Core dump handling is disabled. - if (!environment.getCoredumpFeedEndpoint().isPresent()) return; - - maintainerExecutor.addJob("handle-core-dumps") - .withArgument("coredumpsPath", coredumpsPath) - .withArgument("doneCoredumpsPath", environment.pathInNodeAdminToDoneCoredumps()) - .withArgument("attributes", nodeAttributes) - .withArgument("feedEndpoint", environment.getCoredumpFeedEndpoint().get()); - } + coredumpHandler.processAll(coredumpsPath, nodeAttributes); } private Map<String, Object> getCoredumpNodeAttributes(NodeSpec node) { @@ -354,60 +274,22 @@ public class StorageMaintainer { } /** - * Deletes old - * * archived app data - * * Vespa logs - * * Filedistribution files - */ - public void cleanNodeAdmin() { - if (! getMaintenanceThrottlerFor(NODE_ADMIN).shouldRemoveOldFilesNow()) return; - - MaintainerExecutor maintainerExecutor = new MaintainerExecutor(); - maintainerExecutor.addJob("delete-directories") - .withArgument("basePath", environment.getPathResolver().getApplicationStoragePathForNodeAdmin()) - .withArgument("maxAgeSeconds", Duration.ofDays(7).getSeconds()) - .withArgument("dirNameRegex", "^" + Pattern.quote(Environment.APPLICATION_STORAGE_CLEANUP_PATH_PREFIX)); - - Path nodeAdminJDiskLogsPath = environment.pathInNodeAdminFromPathInNode( - NODE_ADMIN, environment.pathInNodeUnderVespaHome("logs/vespa/")); - maintainerExecutor.addJob("delete-files") - .withArgument("basePath", nodeAdminJDiskLogsPath) - .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds()) - .withArgument("recursive", false); - - Path fileDistrDir = environment.pathInNodeAdminFromPathInNode( - NODE_ADMIN, environment.pathInNodeUnderVespaHome("var/db/vespa/filedistribution")); - maintainerExecutor.addJob("delete-files") - .withArgument("basePath", fileDistrDir) - .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds()) - .withArgument("recursive", true); - - maintainerExecutor.execute(); - getMaintenanceThrottlerFor(NODE_ADMIN).updateNextRemoveOldFilesTime(); - } - - /** - * Prepares the container-storage for the next container by deleting/archiving all the data of the current container. - * Removes old files, reports coredumps and archives container data, runs when container enters state "dirty" + * Prepares the container-storage for the next container by archiving container logs to a new directory + * and deleting everything else owned by this container. */ public void cleanupNodeStorage(ContainerName containerName, NodeSpec node) { - MaintainerExecutor maintainerExecutor = new MaintainerExecutor(); - addRemoveOldFilesCommand(maintainerExecutor, containerName); - addHandleCoredumpsCommand(maintainerExecutor, containerName, node); - addArchiveNodeData(maintainerExecutor, containerName); + removeOldFilesFromNode(containerName); - maintainerExecutor.execute(); - getMaintenanceThrottlerFor(containerName).reset(); - } + Path logsDirInContainer = environment.pathInNodeUnderVespaHome("logs"); + Path containerLogsInArchiveDir = environment.pathInNodeAdminToNodeCleanup(containerName).resolve(logsDirInContainer); + Path containerLogsPathOnHost = environment.pathInNodeAdminFromPathInNode(containerName, logsDirInContainer); - private void addArchiveNodeData(MaintainerExecutor maintainerExecutor, ContainerName containerName) { - maintainerExecutor.addJob("recursive-delete") - .withArgument("path", environment.pathInNodeAdminFromPathInNode( - containerName, environment.pathInNodeUnderVespaHome("var"))); + FileHelper.createDirectories(containerLogsInArchiveDir.getParent()); + FileHelper.moveIfExists(containerLogsPathOnHost, containerLogsInArchiveDir); - maintainerExecutor.addJob("move-files") - .withArgument("from", environment.pathInNodeAdminFromPathInNode(containerName, Paths.get("/"))) - .withArgument("to", environment.pathInNodeAdminToNodeCleanup(containerName)); + FileHelper.streamContents(environment.pathInHostFromPathInNode(containerName, Paths.get("/"))) + .includeBase(true) + .delete(); } /** @@ -452,7 +334,6 @@ public class StorageMaintainer { Pair<Integer, String> result = processExecuter.exec(command); if (result.getFirst() != 0) { - numberOfNodeAdminMaintenanceFails.add(); throw new RuntimeException( String.format("Maintainer failed to execute command: %s, Exit code: %d, Stdout/stderr: %s", Arrays.toString(command), result.getFirst(), result.getSecond())); @@ -462,69 +343,4 @@ public class StorageMaintainer { throw new RuntimeException("Failed to execute maintainer", e); } } - - /** - * Wrapper for node-admin-maintenance, queues up maintenances jobs and sends a single request to maintenance JVM - */ - private class MaintainerExecutor { - private final List<MaintainerExecutorJob> jobs = new ArrayList<>(); - - MaintainerExecutorJob addJob(String jobName) { - MaintainerExecutorJob job = new MaintainerExecutorJob(jobName); - jobs.add(job); - return job; - } - - void execute() { - if (jobs.isEmpty()) return; - - String args; - try { - args = objectMapper.writeValueAsString(jobs); - } catch (JsonProcessingException e) { - throw new RuntimeException("Failed transform list of maintenance jobs to JSON"); - } - - executeMaintainer("com.yahoo.vespa.hosted.node.maintainer.Maintainer", args); - } - } - - private class MaintainerExecutorJob { - @JsonProperty(value="type") - private final String type; - - @JsonProperty(value="arguments") - private final Map<String, Object> arguments = new HashMap<>(); - - MaintainerExecutorJob(String type) { - this.type = type; - } - - MaintainerExecutorJob withArgument(String argument, Object value) { - // Transform Path to String, otherwise ObjectMapper wont encode/decode it properly on the other end - arguments.put(argument, (value instanceof Path) ? value.toString() : value); - return this; - } - } - - private MaintenanceThrottler getMaintenanceThrottlerFor(ContainerName containerName) { - maintenanceThrottlerByContainerName.putIfAbsent(containerName, new MaintenanceThrottler()); - return maintenanceThrottlerByContainerName.get(containerName); - } - - private class MaintenanceThrottler { - private Instant nextRemoveOldFilesAt = Instant.EPOCH; - - void updateNextRemoveOldFilesTime() { - nextRemoveOldFilesAt = clock.instant().plus(Duration.ofHours(1)); - } - - boolean shouldRemoveOldFilesNow() { - return !nextRemoveOldFilesAt.isAfter(clock.instant()); - } - - void reset() { - nextRemoveOldFilesAt = Instant.EPOCH; - } - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index eb48086eb0f..e46b29cc078 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -3,19 +3,18 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.system.ProcessExecuter; -import com.yahoo.vespa.hosted.node.admin.maintenance.FileHelper; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileHelper; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Duration; import java.util.Comparator; import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.regex.Pattern; /** * Finds coredumps, collects metadata and reports them @@ -24,6 +23,7 @@ import java.util.logging.Logger; */ public class CoredumpHandler { + private static final Pattern JAVA_COREDUMP_PATTERN = Pattern.compile("java_pid.*\\.hprof"); private static final String PROCESSING_DIRECTORY_NAME = "processing"; static final String METADATA_FILE_NAME = "metadata.json"; @@ -44,23 +44,11 @@ public class CoredumpHandler { this.doneCoredumpsPath = doneCoredumpsPath; } - public void processAll(Path coredumpsPath, Map<String, Object> nodeAttributes) throws IOException { - removeJavaCoredumps(coredumpsPath); - handleNewCoredumps(coredumpsPath, nodeAttributes); - removeOldCoredumps(); - } - - private void removeJavaCoredumps(Path coredumpsPath) throws IOException { - if (! coredumpsPath.toFile().isDirectory()) return; - FileHelper.deleteFiles(coredumpsPath, Duration.ZERO, Optional.of("^java_pid.*\\.hprof$"), false); - } - - private void removeOldCoredumps() throws IOException { - if (! doneCoredumpsPath.toFile().isDirectory()) return; - FileHelper.deleteDirectories(doneCoredumpsPath, Duration.ofDays(10), Optional.empty()); - } + public void processAll(Path coredumpsPath, Map<String, Object> nodeAttributes) { + FileHelper.streamFiles(coredumpsPath) + .filterFile(FileHelper.nameMatches(JAVA_COREDUMP_PATTERN)) + .delete(); - private void handleNewCoredumps(Path coredumpsPath, Map<String, Object> nodeAttributes) { enqueueCoredumps(coredumpsPath); processAndReportCoredumps(coredumpsPath, nodeAttributes); } @@ -72,12 +60,14 @@ public class CoredumpHandler { */ void enqueueCoredumps(Path coredumpsPath) { Path processingCoredumpsPath = getProcessingCoredumpsPath(coredumpsPath); - processingCoredumpsPath.toFile().mkdirs(); - if (!FileHelper.listContentsOfDirectory(processingCoredumpsPath).isEmpty()) return; - - FileHelper.listContentsOfDirectory(coredumpsPath).stream() - .filter(path -> path.toFile().isFile() && ! path.getFileName().toString().startsWith(".")) - .min((Comparator.comparingLong(o -> o.toFile().lastModified()))) + FileHelper.createDirectories(processingCoredumpsPath); + if (!FileHelper.streamDirectories(processingCoredumpsPath).list().isEmpty()) return; + + FileHelper.streamFiles(coredumpsPath) + .filterFile(FileHelper.nameStartsWith(".").negate()) + .stream() + .min(Comparator.comparing(FileHelper.FileAttributes::lastModifiedTime)) + .map(FileHelper.FileAttributes::path) .ifPresent(coredumpPath -> { try { enqueueCoredumpForProcessing(coredumpPath, processingCoredumpsPath); @@ -89,11 +79,10 @@ public class CoredumpHandler { void processAndReportCoredumps(Path coredumpsPath, Map<String, Object> nodeAttributes) { Path processingCoredumpsPath = getProcessingCoredumpsPath(coredumpsPath); - doneCoredumpsPath.toFile().mkdirs(); + FileHelper.createDirectories(doneCoredumpsPath); - FileHelper.listContentsOfDirectory(processingCoredumpsPath).stream() - .filter(path -> path.toFile().isDirectory()) - .forEach(coredumpDirectory -> processAndReportSingleCoredump(coredumpDirectory, nodeAttributes)); + FileHelper.streamDirectories(processingCoredumpsPath) + .forEachPath(coredumpDirectory -> processAndReportSingleCoredump(coredumpDirectory, nodeAttributes)); } private void processAndReportSingleCoredump(Path coredumpDirectory, Map<String, Object> nodeAttributes) { @@ -109,19 +98,20 @@ public class CoredumpHandler { } private void enqueueCoredumpForProcessing(Path coredumpPath, Path processingCoredumpsPath) throws IOException { - // Make coredump readable - coredumpPath.toFile().setReadable(true, false); - // Create new directory for this coredump and move it into it Path folder = processingCoredumpsPath.resolve(UUID.randomUUID().toString()); - folder.toFile().mkdirs(); + + FileHelper.createDirectories(folder); Files.move(coredumpPath, folder.resolve(coredumpPath.getFileName())); } String collectMetadata(Path coredumpDirectory, Map<String, Object> nodeAttributes) throws IOException { Path metadataPath = coredumpDirectory.resolve(METADATA_FILE_NAME); if (!Files.exists(metadataPath)) { - Path coredumpPath = FileHelper.listContentsOfDirectory(coredumpDirectory).stream().findFirst() + Path coredumpPath = FileHelper.streamFiles(coredumpDirectory) + .stream() + .map(FileHelper.FileAttributes::path) + .findFirst() .orElseThrow(() -> new RuntimeException("No coredump file found in processing directory " + coredumpDirectory)); Map<String, Object> metadata = coreCollector.collect(coredumpPath); metadata.putAll(nodeAttributes); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 2621156487d..519d83bd7d4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; @@ -42,7 +41,6 @@ public class NodeAdminImpl implements NodeAdmin { private final DockerOperations dockerOperations; private final Function<String, NodeAgent> nodeAgentFactory; - private final StorageMaintainer storageMaintainer; private final Runnable aclMaintainer; private final Clock clock; @@ -57,13 +55,11 @@ public class NodeAdminImpl implements NodeAdmin { public NodeAdminImpl(DockerOperations dockerOperations, Function<String, NodeAgent> nodeAgentFactory, - StorageMaintainer storageMaintainer, Runnable aclMaintainer, MetricReceiverWrapper metricReceiver, Clock clock) { this.dockerOperations = dockerOperations; this.nodeAgentFactory = nodeAgentFactory; - this.storageMaintainer = storageMaintainer; this.aclMaintainer = aclMaintainer; this.clock = clock; @@ -82,7 +78,6 @@ public class NodeAdminImpl implements NodeAdmin { .map(NodeSpec::getHostname) .collect(Collectors.toSet()); - storageMaintainer.cleanNodeAdmin(); synchronizeNodesToNodeAgents(hostnamesOfContainersToRun); updateNodeAgentMetrics(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileHelper.java new file mode 100644 index 00000000000..a4b1a66c71b --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileHelper.java @@ -0,0 +1,263 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Stack; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck; + +/** + * @author freva + */ +public class FileHelper { + + private final Path basePath; + private Predicate<FileAttributes> fileFilter; + private Predicate<FileAttributes> directoryFilter; + private boolean includeBase = false; + private boolean recursive = false; + + public FileHelper(Path basePath, boolean includeFiles, boolean includeDirectories) { + this.basePath = basePath; + this.fileFilter = path -> includeFiles; + this.directoryFilter = path -> includeDirectories; + } + + /** + * Creates a {@link FileHelper} that will by default match all files and all directories + * under the given basePath. + */ + public static FileHelper streamContents(Path basePath) { + return new FileHelper(basePath, true, true); + } + + /** + * Creates a {@link FileHelper} that will by default match all files and no directories + * under the given basePath. + */ + public static FileHelper streamFiles(Path basePath) { + return new FileHelper(basePath, true, false); + } + + /** + * Creates a {@link FileHelper} that will by default match all directories and no files + * under the given basePath. + */ + public static FileHelper streamDirectories(Path basePath) { + return new FileHelper(basePath, false, true); + } + + + /** + * Filter that will be used to match files under the base path. Files include everything that + * is not a directory (such as symbolic links) + */ + public FileHelper filterFile(Predicate<FileAttributes> fileFilter) { + this.fileFilter = fileFilter; + return this; + } + + /** + * Filter that will be used to match directories under the base path. + * + * NOTE: When a directory is matched, all of its sub-directories and files are also matched + */ + public FileHelper filterDirectory(Predicate<FileAttributes> directoryFilter) { + this.directoryFilter = directoryFilter; + return this; + } + + /** + * Whether the search should be recursive. + * + * WARNING: When using {@link #delete()} and matching directories, make sure that the directories + * either are already empty or that recursive is set + */ + public FileHelper recursive(boolean recursive) { + this.recursive = recursive; + return this; + } + + /** + * Whether the base path should also be considered (i.e. checked against the correspoding filter). + * When using {@link #delete()} with directories, this is the difference between + * `rm -rf basePath` (true) and `rm -rf basePath/*` (false) + */ + public FileHelper includeBase(boolean includeBase) { + this.includeBase = includeBase; + return this; + } + + public int delete() { + int[] numDeletions = { 0 }; // :( + forEach(attributes -> { + if (deleteIfExists(attributes.path())) + numDeletions[0]++; + }); + + return numDeletions[0]; + } + + public List<FileAttributes> list() { + LinkedList<FileAttributes> list = new LinkedList<>(); + forEach(list::add); + return list; + } + + public Stream<FileAttributes> stream() { + return list().stream(); + } + + public void forEachPath(Consumer<Path> action) { + forEach(attributes -> action.accept(attributes.path())); + } + + /** Applies a given consumer to all the matching {@link FileHelper.FileAttributes} */ + public void forEach(Consumer<FileAttributes> action) { + applyForEachToMatching(basePath, fileFilter, directoryFilter, recursive, includeBase, action); + } + + + /** + * <p> This method walks a file tree rooted at a given starting file. The file tree traversal is + * <em>depth-first</em>: The filter function is applied in pre-order (NLR), but the given + * {@link Consumer} will be called in post-order (LRN). + */ + private void applyForEachToMatching(Path basePath, Predicate<FileAttributes> fileFilter, Predicate<FileAttributes> directoryFilter, + boolean recursive, boolean includeBase, Consumer<FileAttributes> action) { + try { + Files.walkFileTree(basePath, Collections.emptySet(), recursive ? Integer.MAX_VALUE : 1, new SimpleFileVisitor<Path>() { + private Stack<FileAttributes> matchingDirectoryStack = new Stack<>(); + private int currentLevel = -1; + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + currentLevel++; + + FileAttributes attributes = new FileAttributes(dir, attrs); + if (!matchingDirectoryStack.empty() || directoryFilter.test(attributes)) + matchingDirectoryStack.push(attributes); + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + // When we find a directory at the max depth given to Files.walkFileTree, the directory + // will be passed to visitFile() rather than (pre|post)VisitDirectory + if (attrs.isDirectory()) { + preVisitDirectory(file, attrs); + return postVisitDirectory(file, null); + } + + FileAttributes attributes = new FileAttributes(file, attrs); + if (!matchingDirectoryStack.isEmpty() || fileFilter.test(attributes)) + action.accept(attributes); + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + if (!matchingDirectoryStack.isEmpty()) { + FileAttributes attributes = matchingDirectoryStack.pop(); + if (currentLevel != 0 || includeBase) + action.accept(attributes); + } + + currentLevel--; + return FileVisitResult.CONTINUE; + } + }); + } catch (NoSuchFileException ignored) { + + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + + // Ideally, we would reuse the FileAttributes in this package, but unfortunately we only get + // BasicFileAttributes and not PosixFileAttributes from FileVisitor + public static class FileAttributes { + private final Path path; + private final BasicFileAttributes attributes; + + FileAttributes(Path path, BasicFileAttributes attributes) { + this.path = path; + this.attributes = attributes; + } + + public Path path() { return path; } + public String filename() { return path.getFileName().toString(); } + public Instant lastModifiedTime() { return attributes.lastModifiedTime().toInstant(); } + public boolean isRegularFile() { return attributes.isRegularFile(); } + public boolean isDirectory() { return attributes.isDirectory(); } + public long size() { return attributes.size(); } + } + + + // Filters + public static Predicate<FileAttributes> olderThan(Duration duration) { + return attrs -> Duration.between(attrs.lastModifiedTime(), Instant.now()).compareTo(duration) > 0; + } + + public static Predicate<FileAttributes> youngerThan(Duration duration) { + return olderThan(duration).negate(); + } + + public static Predicate<FileAttributes> largerThan(long sizeInBytes) { + return attrs -> attrs.size() > sizeInBytes; + } + + public static Predicate<FileAttributes> smallerThan(long sizeInBytes) { + return largerThan(sizeInBytes).negate(); + } + + public static Predicate<FileAttributes> nameMatches(Pattern pattern) { + return attrs -> pattern.matcher(attrs.filename()).matches(); + } + + public static Predicate<FileAttributes> nameStartsWith(String string) { + return attrs -> attrs.filename().startsWith(string); + } + + public static Predicate<FileAttributes> nameEndsWith(String string) { + return attrs -> attrs.filename().endsWith(string); + } + + + // Other helpful methods that no not throw checked exceptions + public static boolean moveIfExists(Path from, Path to) { + try { + Files.move(from, to); + return true; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static boolean deleteIfExists(Path path) { + return uncheck(() -> Files.deleteIfExists(path)); + } + + public static Path createDirectories(Path path) { + return uncheck(() -> Files.createDirectories(path)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 15bb2825738..49a03c454c1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -87,7 +87,7 @@ public class DockerTester implements AutoCloseable { Clock clock = Clock.systemUTC(); DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment, processExecuter); - StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, null, environment, callOrderVerifier, clock); + StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, null, environment, callOrderVerifier); AclMaintainer aclMaintainer = mock(AclMaintainer.class); AthenzCredentialsMaintainer athenzCredentialsMaintainer = mock(AthenzCredentialsMaintainer.class); @@ -95,7 +95,7 @@ public class DockerTester implements AutoCloseable { MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL, athenzCredentialsMaintainer); - nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); + nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, aclMaintainer, mr, Clock.systemUTC()); nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, storageMaintainer, nodeAdmin, DOCKER_HOST_HOSTNAME, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, Optional.of(new ClassLocking())); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java index a46defc991b..2a3171762bc 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java @@ -245,7 +245,7 @@ public class RunInContainerTest { private final Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, dockerOperationsMock, storageMaintainer, aclMaintainer, environment, Clock.systemUTC(), NODE_AGENT_SCAN_INTERVAL, athenzCredentialsMaintainer); - private final NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperationsMock, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); + private final NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperationsMock, nodeAgentFactory, aclMaintainer, mr, Clock.systemUTC()); private final NodeAdminStateUpdaterImpl nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, storageMaintainer, nodeAdmin, "localhost.test.yahoo.com", Clock.systemUTC(), NODE_ADMIN_CONVERGE_STATE_INTERVAL, Optional.of(new ClassLocking())); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java index 6b7d545c286..62f1a59ecf2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java @@ -1,16 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.component.Environment; -import java.time.Clock; import java.util.Optional; /** @@ -19,8 +16,8 @@ import java.util.Optional; public class StorageMaintainerMock extends StorageMaintainer { private final CallOrderVerifier callOrderVerifier; - public StorageMaintainerMock(DockerOperations dockerOperations, ProcessExecuter processExecuter, Environment environment, CallOrderVerifier callOrderVerifier, Clock clock) { - super(dockerOperations, processExecuter, new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment, clock); + public StorageMaintainerMock(DockerOperations dockerOperations, ProcessExecuter processExecuter, Environment environment, CallOrderVerifier callOrderVerifier) { + super(dockerOperations, processExecuter, environment, null); this.callOrderVerifier = callOrderVerifier; } @@ -38,10 +35,6 @@ public class StorageMaintainerMock extends StorageMaintainer { } @Override - public void cleanNodeAdmin() { - } - - @Override public void cleanupNodeStorage(ContainerName containerName, NodeSpec node) { callOrderVerifier.add("DeleteContainerStorage with " + containerName); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java deleted file mode 100644 index 6b53bc217c4..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; -import java.util.Optional; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author freva - */ -public class FileHelperTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Before - public void initFiles() throws IOException { - for (int i=0; i<10; i++) { - File temp = folder.newFile("test_" + i + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(130).toMillis()); - } - - for (int i=0; i<7; i++) { - File temp = folder.newFile("test_" + i + "_file.test"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(250).toMillis()); - } - - for (int i=0; i<5; i++) { - File temp = folder.newFile(i + "-abc" + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(80).toMillis()); - } - - File temp = folder.newFile("week_old_file.json"); - temp.setLastModified(System.currentTimeMillis() - Duration.ofDays(8).toMillis()); - } - - @Test - public void testDeleteAll() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.empty(), false); - - assertEquals(0, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(6, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file - } - - @Test - public void testDeleteSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".json$"), false); - - assertEquals(7, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefixAndSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 7 test_*_file.test files + week_old_file - } - - @Test - public void testDeleteOld() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(600), Optional.empty(), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 6 (from test_*_.json) - 3 (from test_*_file.test) - 1 week old file - } - - @Test - public void testDeleteWithAllParameters() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(200), Optional.of("^test_.*\\.json$"), false); - - assertEquals(15, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 8 (from test_*_.json) - } - - @Test - public void testDeleteWithSubDirectoriesNoRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - // 6 test_*.json from test_folder1/ - // + 9 test_*.json and 4 abc_*.json from test_folder2/ - // + 13 test_*.json from test_folder2/subSubFolder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder1/ and test_folder2/ and test_folder2/subSubFolder2/ themselves - assertEquals(48, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteWithSubDirectoriesRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), true); - - // 4 abc_*.json from test_folder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder2/ itself - assertEquals(18, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteFilesWhereFilenameRegexAlsoMatchesDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(8, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file + 2 directories - } - - @Test - public void testGetContentsOfNonExistingDirectory() { - Path fakePath = Paths.get("/some/made/up/dir/"); - assertEquals(Collections.emptyList(), FileHelper.listContentsOfDirectory(fakePath)); - } - - @Test(expected=IllegalArgumentException.class) - public void testDeleteFilesExceptNMostRecentWithNegativeN() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), -5); - } - - @Test - public void testDeleteFilesExceptFiveMostRecent() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 5); - - assertEquals(5, getContentsOfDirectory(folder.getRoot()).length); - - String[] oldestFiles = {"test_5_file.test", "test_6_file.test", "test_8.json", "test_9.json", "week_old_file.json"}; - String[] remainingFiles = Arrays.stream(getContentsOfDirectory(folder.getRoot())) - .map(File::getName) - .sorted() - .toArray(String[]::new); - - assertArrayEquals(oldestFiles, remainingFiles); - } - - @Test - public void testDeleteFilesExceptNMostRecentWithLargeN() throws IOException { - String[] filesPreDelete = folder.getRoot().list(); - - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 50); - - assertArrayEquals(filesPreDelete, folder.getRoot().list()); - } - - @Test - public void testDeleteFilesLargerThan10B() throws IOException { - initSubDirectories(); - - File temp1 = new File(folder.getRoot(), "small_file"); - writeNBytesToFile(temp1, 50); - - File temp2 = new File(folder.getRoot(), "some_file"); - writeNBytesToFile(temp2, 20); - - File temp3 = new File(folder.getRoot(), "test_folder1/some_other_file"); - writeNBytesToFile(temp3, 75); - - FileHelper.deleteFilesLargerThan(folder.getRoot().toPath(), 10); - - assertEquals(58, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - assertFalse(temp1.exists() || temp2.exists() || temp3.exists()); - } - - @Test - public void testDeleteDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".*folder2")); - - //23 files in root - // + 6 in test_folder1 + test_folder1 itself - assertEquals(30, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteDirectoriesBasedOnAge() throws IOException { - initSubDirectories(); - // Create folder3 which is older than maxAge, inside have a single directory, subSubFolder3, inside it which is - // also older than maxAge inside the sub directory, create some files which are newer than maxAge. - // deleteDirectories() should NOT delete folder3 - File subFolder3 = folder.newFolder("test_folder3"); - File subSubFolder3 = folder.newFolder("test_folder3", "subSubFolder3"); - - for (int j=0; j<11; j++) { - File.createTempFile("test_", ".json", subSubFolder3); - } - - subFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ofSeconds(50), Optional.of(".*folder.*")); - - //23 files in root - // + 13 in test_folder2 - // + 13 in subSubFolder2 - // + 11 in subSubFolder3 - // + test_folder2 + subSubFolder2 + folder3 + subSubFolder3 itself - assertEquals(64, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testRecursivelyDeleteDirectory() throws IOException { - initSubDirectories(); - FileHelper.recursiveDelete(folder.getRoot().toPath()); - assertFalse(folder.getRoot().exists()); - } - - @Test - public void testRecursivelyDeleteRegularFile() throws IOException { - File file = folder.newFile(); - assertTrue(file.exists()); - assertTrue(file.isFile()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testRecursivelyDeleteNonExistingFile() throws IOException { - File file = folder.getRoot().toPath().resolve("non-existing-file.json").toFile(); - assertFalse(file.exists()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testInitSubDirectories() throws IOException { - initSubDirectories(); - assertTrue(folder.getRoot().exists()); - assertTrue(folder.getRoot().isDirectory()); - - Path test_folder1 = folder.getRoot().toPath().resolve("test_folder1"); - assertTrue(test_folder1.toFile().exists()); - assertTrue(test_folder1.toFile().isDirectory()); - - Path test_folder2 = folder.getRoot().toPath().resolve("test_folder2"); - assertTrue(test_folder2.toFile().exists()); - assertTrue(test_folder2.toFile().isDirectory()); - - Path subSubFolder2 = test_folder2.resolve("subSubFolder2"); - assertTrue(subSubFolder2.toFile().exists()); - assertTrue(subSubFolder2.toFile().isDirectory()); - } - - @Test - public void testDoesNotFailOnLastModifiedOnSymLink() throws IOException { - Path symPath = folder.getRoot().toPath().resolve("symlink"); - Path fakePath = Paths.get("/some/not/existant/file"); - - Files.createSymbolicLink(symPath, fakePath); - assertTrue(Files.isSymbolicLink(symPath)); - assertFalse(Files.exists(fakePath)); - - // Not possible to set modified time on symlink in java, so just check that it doesn't crash - FileHelper.getLastModifiedTime(symPath).toInstant(); - } - - private void initSubDirectories() throws IOException { - File subFolder1 = folder.newFolder("test_folder1"); - File subFolder2 = folder.newFolder("test_folder2"); - File subSubFolder2 = folder.newFolder("test_folder2", "subSubFolder2"); - - for (int j=0; j<6; j++) { - File temp = File.createTempFile("test_", ".json", subFolder1); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(60).toMillis()); - } - - for (int j=0; j<9; j++) { - File.createTempFile("test_", ".json", subFolder2); - } - - for (int j=0; j<4; j++) { - File.createTempFile("abc_", ".txt", subFolder2); - } - - for (int j=0; j<13; j++) { - File temp = File.createTempFile("test_", ".json", subSubFolder2); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(40).toMillis()); - } - - //Must be after all the files have been created - subFolder1.setLastModified(System.currentTimeMillis() - Duration.ofHours(2).toMillis()); - subFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - } - - private static int getNumberOfFilesAndDirectoriesIn(File folder) { - int total = 0; - for (File file : getContentsOfDirectory(folder)) { - if (file.isDirectory()) { - total += getNumberOfFilesAndDirectoriesIn(file); - } - total++; - } - - return total; - } - - private static void writeNBytesToFile(File file, int nBytes) throws IOException { - Files.write(file.toPath(), new byte[nBytes]); - } - - private static File[] getContentsOfDirectory(File directory) { - File[] directoryContents = directory.listFiles(); - - return directoryContents == null ? new File[0] : directoryContents; - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index d9cce7f80a0..7722354a633 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -1,20 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance; -import com.yahoo.collections.Pair; -import com.yahoo.config.provision.NodeType; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.system.ProcessExecuter; -import com.yahoo.test.ManualClock; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.component.PathResolver; -import com.yahoo.vespa.hosted.provision.Node; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -22,21 +14,15 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.nio.file.Files; -import java.time.Duration; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; /** * @author dybis */ public class StorageMaintainerTest { - private final ManualClock clock = new ManualClock(); private final Environment environment = new Environment.Builder() .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder())) .region("us-east-1") @@ -50,7 +36,7 @@ public class StorageMaintainerTest { private final DockerOperations docker = mock(DockerOperations.class); private final ProcessExecuter processExecuter = mock(ProcessExecuter.class); private final StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, - new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment, clock); + environment, null); @Rule public TemporaryFolder folder = new TemporaryFolder(); @@ -71,76 +57,7 @@ public class StorageMaintainerTest { assertEquals(0L, usedBytes); } - @Test - public void testMaintenanceThrottlingAfterSuccessfulMaintenance() { - String hostname = "node-123.us-north-3.test.yahoo.com"; - ContainerName containerName = ContainerName.fromHostname(hostname); - NodeSpec node = new NodeSpec.Builder() - .hostname(hostname) - .state(Node.State.ready) - .nodeType(NodeType.tenant) - .flavor("docker") - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build(); - - try { - when(processExecuter.exec(any(String[].class))).thenReturn(new Pair<>(0, "")); - } catch (IOException ignored) { } - storageMaintainer.removeOldFilesFromNode(containerName); - verifyProcessExecuterCalled(1); - // Will not actually run maintenance job until an hour passes - storageMaintainer.removeOldFilesFromNode(containerName); - verifyProcessExecuterCalled(1); - - clock.advance(Duration.ofMinutes(61)); - storageMaintainer.removeOldFilesFromNode(containerName); - verifyProcessExecuterCalled(2); - - // Coredump handling is unthrottled - storageMaintainer.handleCoreDumpsForContainer(containerName, node); - verifyProcessExecuterCalled(3); - - storageMaintainer.handleCoreDumpsForContainer(containerName, node); - verifyProcessExecuterCalled(4); - - // cleanupNodeStorage is unthrottled and it should reset previous times - storageMaintainer.cleanupNodeStorage(containerName, node); - verifyProcessExecuterCalled(5); - storageMaintainer.cleanupNodeStorage(containerName, node); - verifyProcessExecuterCalled(6); - } - - @Test - public void testMaintenanceThrottlingAfterFailedMaintenance() { - String hostname = "node-123.us-north-3.test.yahoo.com"; - ContainerName containerName = ContainerName.fromHostname(hostname); - - try { - when(processExecuter.exec(any(String[].class))) - .thenThrow(new RuntimeException("Something went wrong")) - .thenReturn(new Pair<>(0, "")); - } catch (IOException ignored) { } - - try { - storageMaintainer.removeOldFilesFromNode(containerName); - fail("Maintenance job should've failed!"); - } catch (RuntimeException ignored) { } - verifyProcessExecuterCalled(1); - - // Maintenance job failed, we should be able to immediately re-run it - storageMaintainer.removeOldFilesFromNode(containerName); - verifyProcessExecuterCalled(2); - } - private static void writeNBytesToFile(File file, int nBytes) throws IOException { Files.write(file.toPath(), new byte[nBytes]); } - - private void verifyProcessExecuterCalled(int times) { - try { - verify(processExecuter, times(times)).exec(any(String[].class)); - } catch (IOException ignored) { } - } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index efd35cce00e..065b039c7fd 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -5,7 +5,6 @@ import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import org.junit.Test; @@ -40,11 +39,10 @@ public class NodeAdminImplTest { private interface NodeAgentFactory extends Function<String, NodeAgent> {} private final DockerOperations dockerOperations = mock(DockerOperations.class); private final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); - private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); private final Runnable aclMaintainer = mock(Runnable.class); private final ManualClock clock = new ManualClock(); - private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, + private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, aclMaintainer, new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock); @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileHelperTest.java new file mode 100644 index 00000000000..a3569853122 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileHelperTest.java @@ -0,0 +1,201 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author freva + */ +@RunWith(Enclosed.class) +public class FileHelperTest { + + public static class GeneralLogicTests { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void delete_all_files_non_recursive() { + int numDeleted = FileHelper.streamFiles(testRoot()) + .delete(); + + assertEquals(3, numDeleted); + assertRecursiveContents("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/file", "test/subdir-2"); + } + + @Test + public void delete_all_files_recursive() { + int numDeleted = FileHelper.streamFiles(testRoot()) + .recursive(true) + .delete(); + + assertEquals(6, numDeleted); + assertRecursiveContents("test", "test/subdir-1", "test/subdir-2"); + } + + @Test + public void delete_with_filter_recursive() { + int numDeleted = FileHelper.streamFiles(testRoot()) + .filterFile(FileHelper.nameEndsWith(".json")) + .recursive(true) + .delete(); + + assertEquals(3, numDeleted); + assertRecursiveContents("test.txt", "test", "test/file.txt", "test/subdir-1", "test/subdir-1/file", "test/subdir-2"); + } + + @Test + public void delete_directory_with_filter() { + int numDeleted = FileHelper.streamDirectories(testRoot()) + .filterDirectory(FileHelper.nameStartsWith("subdir")) + .recursive(true) + .delete(); + + assertEquals(3, numDeleted); + assertRecursiveContents("file-1.json", "test.json", "test.txt", "test", "test/file.txt", "test/data.json"); + } + + @Test + public void delete_all_contents() { + int numDeleted = FileHelper.streamContents(testRoot()) + .recursive(true) + .delete(); + + assertEquals(9, numDeleted); + assertTrue(Files.exists(testRoot())); + assertRecursiveContents(); + } + + @Test + public void delete_everything() { + int numDeleted = FileHelper.streamContents(testRoot()) + .includeBase(true) + .recursive(true) + .delete(); + + assertEquals(10, numDeleted); + assertFalse(Files.exists(testRoot())); + } + + @Before + public void setup() throws IOException { + Path root = testRoot(); + + Files.createFile(root.resolve("file-1.json")); + Files.createFile(root.resolve("test.json")); + Files.createFile(root.resolve("test.txt")); + + Files.createDirectories(root.resolve("test")); + Files.createFile(root.resolve("test/file.txt")); + Files.createFile(root.resolve("test/data.json")); + + Files.createDirectories(root.resolve("test/subdir-1")); + Files.createFile(root.resolve("test/subdir-1/file")); + + Files.createDirectories(root.resolve("test/subdir-2")); + } + + private Path testRoot() { + return folder.getRoot().toPath(); + } + + private void assertRecursiveContents(String... relativePaths) { + Set<String> expectedPaths = new HashSet<>(Arrays.asList(relativePaths)); + Set<String> actualPaths = recursivelyListContents(testRoot()).stream() + .map(testRoot()::relativize) + .map(Path::toString) + .collect(Collectors.toSet()); + + assertEquals(expectedPaths, actualPaths); + } + + private List<Path> recursivelyListContents(Path basePath) { + try (Stream<Path> pathStream = Files.list(basePath)) { + List<Path> paths = new LinkedList<>(); + pathStream.forEach(path -> { + paths.add(path); + if (Files.isDirectory(path)) + paths.addAll(recursivelyListContents(path)); + }); + return paths; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + public static class FilterUnitTests { + + private final BasicFileAttributes attributes = mock(BasicFileAttributes.class); + + @Test + public void age_filter_test() { + Path path = Paths.get("/my/fake/path"); + when(attributes.lastModifiedTime()).thenReturn(FileTime.from(Instant.now().minus(Duration.ofHours(1)))); + FileHelper.FileAttributes fileAttributes = new FileHelper.FileAttributes(path, attributes); + + assertFalse(FileHelper.olderThan(Duration.ofMinutes(61)).test(fileAttributes)); + assertTrue(FileHelper.olderThan(Duration.ofMinutes(59)).test(fileAttributes)); + + assertTrue(FileHelper.youngerThan(Duration.ofMinutes(61)).test(fileAttributes)); + assertFalse(FileHelper.youngerThan(Duration.ofMinutes(59)).test(fileAttributes)); + } + + @Test + public void size_filters() { + Path path = Paths.get("/my/fake/path"); + when(attributes.size()).thenReturn(100L); + FileHelper.FileAttributes fileAttributes = new FileHelper.FileAttributes(path, attributes); + + assertFalse(FileHelper.largerThan(101).test(fileAttributes)); + assertTrue(FileHelper.largerThan(99).test(fileAttributes)); + + assertTrue(FileHelper.smallerThan(101).test(fileAttributes)); + assertFalse(FileHelper.smallerThan(99).test(fileAttributes)); + } + + @Test + public void filename_filters() { + Path path = Paths.get("/my/fake/path/some-12352-file.json"); + FileHelper.FileAttributes fileAttributes = new FileHelper.FileAttributes(path, attributes); + + assertTrue(FileHelper.nameStartsWith("some-").test(fileAttributes)); + assertFalse(FileHelper.nameStartsWith("som-").test(fileAttributes)); + + assertTrue(FileHelper.nameEndsWith(".json").test(fileAttributes)); + assertFalse(FileHelper.nameEndsWith("file").test(fileAttributes)); + + assertTrue(FileHelper.nameMatches(Pattern.compile("some-[0-9]+-file.json")).test(fileAttributes)); + assertTrue(FileHelper.nameMatches(Pattern.compile("^some-[0-9]+-file.json$")).test(fileAttributes)); + assertFalse(FileHelper.nameMatches(Pattern.compile("some-[0-9]-file.json")).test(fileAttributes)); + } + } +} |