add a daemon thread doing regular maintenance

* also, refactor test implementation

3 files changed, 123 insertions, 65 deletions

diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
index fbc4a6fb6bb..c6e5c3c372b 100644
--- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
@@ -37,19 +37,36 @@ public class FilesArchived {
     private long maxAgeDays = 30; // GDPR rules: max 30 days
     private long sizeLimit = 30L * (1L << 30); // 30 GB
 
+    private void run() {
+        try {
+            Thread.sleep(125000); // 2 m 5 s
+            while (true) {
+                while (maintenance()) {
+                    Thread.sleep(2000); // 2 s
+                }
+                Thread.sleep(299000); // approx 5 min
+            }
+        } catch (InterruptedException e) {
+            // just exit thread on interrupt
+        }
+    }
+
     /**
-     * Creates an FilesArchive managing the given directory
+     * Creates an instance of FilesArchive managing the given directory
      */
     public FilesArchived(File rootDir) {
         this.root = rootDir;
-        maintenance();
+        rescan();
+        Thread thread = new Thread(this::run);
+        thread.setDaemon(true);
+        thread.start();
     }
 
     public String toString() {
         return FilesArchived.class.getName() + ": root=" + root;
     }
 
-    public int highestGen(String prefix) {
+    public synchronized int highestGen(String prefix) {
         int gen = 0;
         for (LogFile lf : knownFiles) {
             if (prefix.equals(lf.prefix)) {
@@ -59,14 +76,25 @@ public class FilesArchived {
         return gen;
     }
 
-    public synchronized void maintenance() {
+    public synchronized boolean maintenance() {
+        boolean action = false;
         rescan();
-        if (removeOlderThan(maxAgeDays)) rescan();
-        if (compressOldFiles()) rescan();
+        if (removeOlderThan(maxAgeDays)) {
+            action = true;
+            rescan();
+        }
+        if (compressOldFiles()) {
+            action = true;
+            rescan();
+        }
         long days = maxAgeDays;
         while (tooMuchDiskUsage() && (--days > 1)) {
-            if (removeOlderThan(days)) rescan();
+            if (removeOlderThan(days)) {
+                action = true;
+                rescan();
+            }
         }
+        return action;
     }
 
     private void rescan() {
@@ -133,7 +161,7 @@ public class FilesArchived {
         }
     }
 
-    public long sumFileSizes() {
+    long sumFileSizes() {
         long sum = 0;
         for (LogFile lf : knownFiles) {
             sum += lf.path.length();
@@ -222,4 +250,3 @@ public class FilesArchived {
         }
     }
 }
-
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java
index 1c6b3e2a6ad..5490a9fd57f 100644
--- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java
@@ -203,6 +203,8 @@ public class ArchiverHandlerTestCase {
 
     @Test
     public void testCacheEldestEntry() throws IOException {
+        try {
+
         LogWriterLRUCache cache = new LogWriterLRUCache(5, (float) 0.75);
         String d = "target/tmp/logarchive";
         FilesArchived archive = new FilesArchived(new File(d));
@@ -210,6 +212,11 @@ public class ArchiverHandlerTestCase {
             cache.put(i, new LogWriter(d+"/2018/12/31/17", 5, archive));
         }
         assertEquals(cache.size(), cache.maxEntries);
+
+        } catch (NullPointerException e) {
+            System.err.println("null: "+e);
+            e.printStackTrace();
+        }
     }
 
     @Test
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java
index b04354d0b67..313da9a6a83 100644
--- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java
@@ -23,80 +23,104 @@ public class FilesArchivedTestCase {
     @Rule
     public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
-    private void makeLogfile(File dir, String name, long hours) throws IOException {
-        File f = new File(dir, name);
+    File tmpDir;
+
+    private void makeLogfile(String name, long hours) throws IOException {
+        File f = new File(tmpDir, name);
         f.getParentFile().mkdirs();
         new FileWriter(f).write("foo bar baz\n");
         long now = System.currentTimeMillis();
         f.setLastModified(now - (hours * 3600 * 1000));
     }
 
-    void checkExist(File dir, String name) {
-        assertTrue(new File(dir, name).isFile());
+    void checkExist(String name) {
+        assertTrue(new File(tmpDir, name).isFile());
     }
-    void checkNoExist(File dir, String name) {
-        assertFalse(new File(dir, name).isFile());
+    void checkNoExist(String name) {
+        assertFalse(new File(tmpDir, name).isFile());
     }
 
     @Test
     public void testMaintenance() throws java.io.IOException {
-        File tmpDir = temporaryFolder.newFolder();
-
-        makeLogfile(tmpDir, "foo/bar", 35*24); // non-matching file
-
-        makeLogfile(tmpDir, "2018/11/20/13-0", 35*24);
-        makeLogfile(tmpDir, "2018/11/21/13-0", 34*24);
-        makeLogfile(tmpDir, "2018/12/28/13-0", 3*24);
-        makeLogfile(tmpDir, "2018/12/29/13-0", 2*24);
-        makeLogfile(tmpDir, "2018/12/30/13-0", 1*24);
-        makeLogfile(tmpDir, "2018/12/31/14-0", 3);
-        makeLogfile(tmpDir, "2018/12/31/16-0", 1);
-        makeLogfile(tmpDir, "2018/12/31/17-0", 0);
-        dumpFiles(tmpDir, "before archive maintenance");
+        tmpDir = temporaryFolder.newFolder();
+
+        makeLogfile("foo/bar", 35*24); // non-matching file
+
+        makeLogfile("2018/11/20/13-0", 35*24);
+        makeLogfile("2018/11/21/13-0", 34*24);
+        makeLogfile("2018/12/28/13-0", 3*24);
+        makeLogfile("2018/12/29/13-0", 2*24);
+        makeLogfile("2018/12/30/13-0", 1*24);
+        makeLogfile("2018/12/31/14-0", 3);
+        makeLogfile("2018/12/31/16-0", 1);
+        makeLogfile("2018/12/31/17-0", 0);
+        dumpFiles("before archive maintenance");
         FilesArchived a = new FilesArchived(tmpDir);
-        dumpFiles(tmpDir, "after archive maintenance");
-        checkExist(tmpDir, "foo/bar");
-        checkExist(tmpDir, "2018/12/31/17-0");
-        checkExist(tmpDir, "2018/12/31/16-0");
-        checkExist(tmpDir, "2018/12/31/14-0.gz");
-        checkExist(tmpDir, "2018/12/28/13-0.gz");
-        checkExist(tmpDir, "2018/12/29/13-0.gz");
-        checkExist(tmpDir, "2018/12/30/13-0.gz");
-
-        checkNoExist(tmpDir, "2018/12/31/17-0.gz");
-        checkNoExist(tmpDir, "2018/12/31/16-0.gz");
-        checkNoExist(tmpDir, "2018/12/31/14-0");
-        checkNoExist(tmpDir, "2018/12/28/13-0");
-        checkNoExist(tmpDir, "2018/12/29/13-0");
-        checkNoExist(tmpDir, "2018/12/30/13-0");
-
-        checkNoExist(tmpDir, "2018/11/20/13-0");
-        checkNoExist(tmpDir, "2018/11/20/13-0.gz");
-        checkNoExist(tmpDir, "2018/11/21/13-0");
-        checkNoExist(tmpDir, "2018/11/21/13-0.gz");
-
-        makeLogfile(tmpDir, "2018/12/31/16-0", 3);
-        makeLogfile(tmpDir, "2018/12/31/17-0", 3);
-        makeLogfile(tmpDir, "2018/12/31/17-1", 1);
-        makeLogfile(tmpDir, "2018/12/31/17-2", 0);
-
-        dumpFiles(tmpDir, "before second archive maintenance");
+        dumpFiles("also before archive maintenance");
+
+        checkExist("foo/bar");
+        checkExist("2018/11/20/13-0");
+        checkExist("2018/11/21/13-0");
+        checkExist("2018/12/28/13-0");
+        checkExist("2018/12/29/13-0");
+        checkExist("2018/12/30/13-0");
+        checkExist("2018/12/31/14-0");
+        checkExist("2018/12/31/16-0");
+        checkExist("2018/12/31/17-0");
+        checkNoExist("2018/11/20/13-0.gz");
+        checkNoExist("2018/11/21/13-0.gz");
+        checkNoExist("2018/12/28/13-0.gz");
+        checkNoExist("2018/12/29/13-0.gz");
+        checkNoExist("2018/12/30/13-0.gz");
+        checkNoExist("2018/12/31/14-0.gz");
+        checkNoExist("2018/12/31/16-0.gz");
+        checkNoExist("2018/12/31/17-0.gz");
+
+        a.maintenance();
+
+        dumpFiles("after archive maintenance");
+        checkExist("foo/bar");
+        checkExist("2018/12/31/17-0");
+        checkExist("2018/12/31/16-0");
+        checkExist("2018/12/31/14-0.gz");
+        checkExist("2018/12/28/13-0.gz");
+        checkExist("2018/12/29/13-0.gz");
+        checkExist("2018/12/30/13-0.gz");
+
+        checkNoExist("2018/12/31/17-0.gz");
+        checkNoExist("2018/12/31/16-0.gz");
+        checkNoExist("2018/12/31/14-0");
+        checkNoExist("2018/12/28/13-0");
+        checkNoExist("2018/12/29/13-0");
+        checkNoExist("2018/12/30/13-0");
+
+        checkNoExist("2018/11/20/13-0");
+        checkNoExist("2018/11/20/13-0.gz");
+        checkNoExist("2018/11/21/13-0");
+        checkNoExist("2018/11/21/13-0.gz");
+
+        makeLogfile("2018/12/31/16-0", 3);
+        makeLogfile("2018/12/31/17-0", 3);
+        makeLogfile("2018/12/31/17-1", 1);
+        makeLogfile("2018/12/31/17-2", 0);
+
+        dumpFiles("before second archive maintenance");
         a.maintenance();
-        dumpFiles(tmpDir, "after second archive maintenance");
+        dumpFiles("after second archive maintenance");
 
-        checkExist(tmpDir, "2018/12/31/17-2");
-        checkExist(tmpDir, "2018/12/31/17-1");
-        checkExist(tmpDir, "2018/12/31/16-0.gz");
-        checkExist(tmpDir, "2018/12/31/17-0.gz");
+        checkExist("2018/12/31/17-2");
+        checkExist("2018/12/31/17-1");
+        checkExist("2018/12/31/16-0.gz");
+        checkExist("2018/12/31/17-0.gz");
 
-        checkNoExist(tmpDir, "2018/12/31/16-0");
-        checkNoExist(tmpDir, "2018/12/31/17-0");
-        checkExist(tmpDir, "foo/bar");
+        checkNoExist("2018/12/31/16-0");
+        checkNoExist("2018/12/31/17-0");
+        checkExist("foo/bar");
     }
 
-    private void dumpFiles(File dir, String header) {
+    private void dumpFiles(String header) {
         System.out.println(">>> " + header + " >>> :");
-        List<String> seen = scanDir(dir);
+        List<String> seen = scanDir(tmpDir);
         seen.sort(null);
         for (String s : seen) {
             System.err.println("   " + s);