Limit to one coredump processing

2 files changed, 45 insertions, 8 deletions

diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java
index 21584aee18d..53b3b8c7e77 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java
@@ -15,6 +15,7 @@ import java.io.InputStreamReader;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -28,10 +29,10 @@ import java.util.stream.Collectors;
  *
  * @author freva
  */
-public class CoredumpHandler {
+class CoredumpHandler {
 
-    public static final String PROCESSING_DIRECTORY_NAME = "processing";
-    public static final String METADATA_FILE_NAME = "metadata.json";
+    static final String PROCESSING_DIRECTORY_NAME = "processing";
+    static final String METADATA_FILE_NAME = "metadata.json";
 
     private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName());
     private final ObjectMapper objectMapper = new ObjectMapper();
@@ -77,13 +78,19 @@ public class CoredumpHandler {
     }
 
 
+    /**
+     * Moves a coredump to a new directory under the processing/ directory. Limit to only processing
+     * one coredump at the time, starting with the oldest.
+     */
     Path processCoredumps() throws IOException {
         Path processingCoredumpsPath = coredumpsPath.resolve(PROCESSING_DIRECTORY_NAME);
         processingCoredumpsPath.toFile().mkdirs();
+        if (Files.list(processingCoredumpsPath).count() > 0) return processingCoredumpsPath;
 
         Files.list(coredumpsPath)
                 .filter(path -> path.toFile().isFile() && ! path.getFileName().toString().startsWith("."))
-                .forEach(coredumpPath -> {
+                .min((Comparator.comparingLong(o -> o.toFile().lastModified())))
+                .ifPresent(coredumpPath -> {
                     try {
                         startProcessing(coredumpPath, processingCoredumpsPath);
                     } catch (Throwable e) {
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java
index d469e5efc46..f1a28ac84bb 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java
@@ -22,7 +22,9 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -84,7 +86,7 @@ public class CoredumpHandlerTest {
 
     @Test
     public void ignoresIncompleteCoredumps() throws IOException {
-        Path coredumpPath = createCoredump(".core.dump");
+        Path coredumpPath = createCoredump(".core.dump", Instant.now());
         Path processingPath = coredumpHandler.processCoredumps();
 
         // The 'processing' directory should be empty
@@ -96,7 +98,7 @@ public class CoredumpHandlerTest {
 
     @Test
     public void startProcessingTest() throws IOException {
-        Path coredumpPath = createCoredump("core.dump");
+        Path coredumpPath = createCoredump("core.dump", Instant.now());
         Path processingPath = crashPath.resolve("processing_dir");
         coredumpHandler.startProcessing(coredumpPath, processingPath);
 
@@ -112,8 +114,35 @@ public class CoredumpHandlerTest {
     }
 
     @Test
+    public void limitToProcessingOneCoredumpAtTheTimeTest() throws IOException {
+        final String oldestCoredump = "core.dump0";
+        final Instant startTime = Instant.now();
+        createCoredump(oldestCoredump, startTime.minusSeconds(3600));
+        createCoredump("core.dump1", startTime.minusSeconds(1000));
+        createCoredump("core.dump2", startTime);
+        Path processingPath = coredumpHandler.processCoredumps();
+
+        List<Path> processingCoredumps = Files.list(processingPath).collect(Collectors.toList());
+        assertEquals(1, processingCoredumps.size());
+
+        // Make sure that the 1 coredump that we are processing is the oldest one
+        Set<String> filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0))
+                .map(file -> file.getFileName().toString())
+                .collect(Collectors.toSet());
+        assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory);
+
+        // Running processCoredumps should not start processing any new coredumps as we already are processing one
+        coredumpHandler.processCoredumps();
+        assertEquals(processingCoredumps, Files.list(processingPath).collect(Collectors.toList()));
+        filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0))
+                .map(file -> file.getFileName().toString())
+                .collect(Collectors.toSet());
+        assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory);
+    }
+
+    @Test
     public void coredumpMetadataCollectAndWriteTest() throws IOException, InterruptedException {
-        createCoredump("core.dump");
+        createCoredump("core.dump", Instant.now());
         Path processingPath = coredumpHandler.processCoredumps();
         Path processingCoredumpPath = Files.list(processingPath).findFirst().orElseThrow(() ->
                 new RuntimeException("Expected to find directory with coredump in processing dir"));
@@ -182,9 +211,10 @@ public class CoredumpHandlerTest {
         assertEquals(expectedContentsOfFolder, actualContentsOfFolder);
     }
 
-    private Path createCoredump(String coredumpName) throws IOException {
+    private Path createCoredump(String coredumpName, Instant lastModified) throws IOException {
         Path coredumpPath = crashPath.resolve(coredumpName);
         coredumpPath.toFile().createNewFile();
+        coredumpPath.toFile().setLastModified(lastModified.toEpochMilli());
         return coredumpPath;
     }