From 7316cc20b9ebe4fb491c80f6ab1359acba3b0a1c Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 27 Sep 2017 10:05:33 +0200 Subject: Limit to one coredump processing --- .../hosted/node/maintainer/CoredumpHandler.java | 15 ++++++--- .../node/maintainer/CoredumpHandlerTest.java | 38 +++++++++++++++++++--- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java index 21584aee18d..53b3b8c7e77 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java @@ -15,6 +15,7 @@ import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -28,10 +29,10 @@ import java.util.stream.Collectors; * * @author freva */ -public class CoredumpHandler { +class CoredumpHandler { - public static final String PROCESSING_DIRECTORY_NAME = "processing"; - public static final String METADATA_FILE_NAME = "metadata.json"; + static final String PROCESSING_DIRECTORY_NAME = "processing"; + static final String METADATA_FILE_NAME = "metadata.json"; private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName()); private final ObjectMapper objectMapper = new ObjectMapper(); @@ -77,13 +78,19 @@ public class CoredumpHandler { } + /** + * Moves a coredump to a new directory under the processing/ directory. Limit to only processing + * one coredump at the time, starting with the oldest. + */ Path processCoredumps() throws IOException { Path processingCoredumpsPath = coredumpsPath.resolve(PROCESSING_DIRECTORY_NAME); processingCoredumpsPath.toFile().mkdirs(); + if (Files.list(processingCoredumpsPath).count() > 0) return processingCoredumpsPath; Files.list(coredumpsPath) .filter(path -> path.toFile().isFile() && ! path.getFileName().toString().startsWith(".")) - .forEach(coredumpPath -> { + .min((Comparator.comparingLong(o -> o.toFile().lastModified()))) + .ifPresent(coredumpPath -> { try { startProcessing(coredumpPath, processingCoredumpsPath); } catch (Throwable e) { diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java index d469e5efc46..f1a28ac84bb 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java @@ -22,7 +22,9 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; import java.util.Arrays; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -84,7 +86,7 @@ public class CoredumpHandlerTest { @Test public void ignoresIncompleteCoredumps() throws IOException { - Path coredumpPath = createCoredump(".core.dump"); + Path coredumpPath = createCoredump(".core.dump", Instant.now()); Path processingPath = coredumpHandler.processCoredumps(); // The 'processing' directory should be empty @@ -96,7 +98,7 @@ public class CoredumpHandlerTest { @Test public void startProcessingTest() throws IOException { - Path coredumpPath = createCoredump("core.dump"); + Path coredumpPath = createCoredump("core.dump", Instant.now()); Path processingPath = crashPath.resolve("processing_dir"); coredumpHandler.startProcessing(coredumpPath, processingPath); @@ -111,9 +113,36 @@ public class CoredumpHandlerTest { assertFolderContents(processedCoredumps.get(0), coredumpPath.getFileName().toString()); } + @Test + public void limitToProcessingOneCoredumpAtTheTimeTest() throws IOException { + final String oldestCoredump = "core.dump0"; + final Instant startTime = Instant.now(); + createCoredump(oldestCoredump, startTime.minusSeconds(3600)); + createCoredump("core.dump1", startTime.minusSeconds(1000)); + createCoredump("core.dump2", startTime); + Path processingPath = coredumpHandler.processCoredumps(); + + List processingCoredumps = Files.list(processingPath).collect(Collectors.toList()); + assertEquals(1, processingCoredumps.size()); + + // Make sure that the 1 coredump that we are processing is the oldest one + Set filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0)) + .map(file -> file.getFileName().toString()) + .collect(Collectors.toSet()); + assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory); + + // Running processCoredumps should not start processing any new coredumps as we already are processing one + coredumpHandler.processCoredumps(); + assertEquals(processingCoredumps, Files.list(processingPath).collect(Collectors.toList())); + filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0)) + .map(file -> file.getFileName().toString()) + .collect(Collectors.toSet()); + assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory); + } + @Test public void coredumpMetadataCollectAndWriteTest() throws IOException, InterruptedException { - createCoredump("core.dump"); + createCoredump("core.dump", Instant.now()); Path processingPath = coredumpHandler.processCoredumps(); Path processingCoredumpPath = Files.list(processingPath).findFirst().orElseThrow(() -> new RuntimeException("Expected to find directory with coredump in processing dir")); @@ -182,9 +211,10 @@ public class CoredumpHandlerTest { assertEquals(expectedContentsOfFolder, actualContentsOfFolder); } - private Path createCoredump(String coredumpName) throws IOException { + private Path createCoredump(String coredumpName, Instant lastModified) throws IOException { Path coredumpPath = crashPath.resolve(coredumpName); coredumpPath.toFile().createNewFile(); + coredumpPath.toFile().setLastModified(lastModified.toEpochMilli()); return coredumpPath; } -- cgit v1.2.3