From 980abeff7aabe62a4f8d8c5fe314d3b654f8fdad Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 3 Jun 2019 13:50:26 +0000 Subject: add a daemon thread doing regular maintenance * also, refactor test implementation --- .../logserver/handlers/archive/FilesArchived.java | 45 +++++-- .../handlers/archive/ArchiverHandlerTestCase.java | 7 ++ .../handlers/archive/FilesArchivedTestCase.java | 136 ++++++++++++--------- 3 files changed, 123 insertions(+), 65 deletions(-) diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java index fbc4a6fb6bb..c6e5c3c372b 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java @@ -37,19 +37,36 @@ public class FilesArchived { private long maxAgeDays = 30; // GDPR rules: max 30 days private long sizeLimit = 30L * (1L << 30); // 30 GB + private void run() { + try { + Thread.sleep(125000); // 2 m 5 s + while (true) { + while (maintenance()) { + Thread.sleep(2000); // 2 s + } + Thread.sleep(299000); // approx 5 min + } + } catch (InterruptedException e) { + // just exit thread on interrupt + } + } + /** - * Creates an FilesArchive managing the given directory + * Creates an instance of FilesArchive managing the given directory */ public FilesArchived(File rootDir) { this.root = rootDir; - maintenance(); + rescan(); + Thread thread = new Thread(this::run); + thread.setDaemon(true); + thread.start(); } public String toString() { return FilesArchived.class.getName() + ": root=" + root; } - public int highestGen(String prefix) { + public synchronized int highestGen(String prefix) { int gen = 0; for (LogFile lf : knownFiles) { if (prefix.equals(lf.prefix)) { @@ -59,14 +76,25 @@ public class FilesArchived { return gen; } - public synchronized void maintenance() { + public synchronized boolean maintenance() { + boolean action = false; rescan(); - if (removeOlderThan(maxAgeDays)) rescan(); - if (compressOldFiles()) rescan(); + if (removeOlderThan(maxAgeDays)) { + action = true; + rescan(); + } + if (compressOldFiles()) { + action = true; + rescan(); + } long days = maxAgeDays; while (tooMuchDiskUsage() && (--days > 1)) { - if (removeOlderThan(days)) rescan(); + if (removeOlderThan(days)) { + action = true; + rescan(); + } } + return action; } private void rescan() { @@ -133,7 +161,7 @@ public class FilesArchived { } } - public long sumFileSizes() { + long sumFileSizes() { long sum = 0; for (LogFile lf : knownFiles) { sum += lf.path.length(); @@ -222,4 +250,3 @@ public class FilesArchived { } } } - diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java index 1c6b3e2a6ad..5490a9fd57f 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java @@ -203,6 +203,8 @@ public class ArchiverHandlerTestCase { @Test public void testCacheEldestEntry() throws IOException { + try { + LogWriterLRUCache cache = new LogWriterLRUCache(5, (float) 0.75); String d = "target/tmp/logarchive"; FilesArchived archive = new FilesArchived(new File(d)); @@ -210,6 +212,11 @@ public class ArchiverHandlerTestCase { cache.put(i, new LogWriter(d+"/2018/12/31/17", 5, archive)); } assertEquals(cache.size(), cache.maxEntries); + + } catch (NullPointerException e) { + System.err.println("null: "+e); + e.printStackTrace(); + } } @Test diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java index b04354d0b67..313da9a6a83 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java @@ -23,80 +23,104 @@ public class FilesArchivedTestCase { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - private void makeLogfile(File dir, String name, long hours) throws IOException { - File f = new File(dir, name); + File tmpDir; + + private void makeLogfile(String name, long hours) throws IOException { + File f = new File(tmpDir, name); f.getParentFile().mkdirs(); new FileWriter(f).write("foo bar baz\n"); long now = System.currentTimeMillis(); f.setLastModified(now - (hours * 3600 * 1000)); } - void checkExist(File dir, String name) { - assertTrue(new File(dir, name).isFile()); + void checkExist(String name) { + assertTrue(new File(tmpDir, name).isFile()); } - void checkNoExist(File dir, String name) { - assertFalse(new File(dir, name).isFile()); + void checkNoExist(String name) { + assertFalse(new File(tmpDir, name).isFile()); } @Test public void testMaintenance() throws java.io.IOException { - File tmpDir = temporaryFolder.newFolder(); - - makeLogfile(tmpDir, "foo/bar", 35*24); // non-matching file - - makeLogfile(tmpDir, "2018/11/20/13-0", 35*24); - makeLogfile(tmpDir, "2018/11/21/13-0", 34*24); - makeLogfile(tmpDir, "2018/12/28/13-0", 3*24); - makeLogfile(tmpDir, "2018/12/29/13-0", 2*24); - makeLogfile(tmpDir, "2018/12/30/13-0", 1*24); - makeLogfile(tmpDir, "2018/12/31/14-0", 3); - makeLogfile(tmpDir, "2018/12/31/16-0", 1); - makeLogfile(tmpDir, "2018/12/31/17-0", 0); - dumpFiles(tmpDir, "before archive maintenance"); + tmpDir = temporaryFolder.newFolder(); + + makeLogfile("foo/bar", 35*24); // non-matching file + + makeLogfile("2018/11/20/13-0", 35*24); + makeLogfile("2018/11/21/13-0", 34*24); + makeLogfile("2018/12/28/13-0", 3*24); + makeLogfile("2018/12/29/13-0", 2*24); + makeLogfile("2018/12/30/13-0", 1*24); + makeLogfile("2018/12/31/14-0", 3); + makeLogfile("2018/12/31/16-0", 1); + makeLogfile("2018/12/31/17-0", 0); + dumpFiles("before archive maintenance"); FilesArchived a = new FilesArchived(tmpDir); - dumpFiles(tmpDir, "after archive maintenance"); - checkExist(tmpDir, "foo/bar"); - checkExist(tmpDir, "2018/12/31/17-0"); - checkExist(tmpDir, "2018/12/31/16-0"); - checkExist(tmpDir, "2018/12/31/14-0.gz"); - checkExist(tmpDir, "2018/12/28/13-0.gz"); - checkExist(tmpDir, "2018/12/29/13-0.gz"); - checkExist(tmpDir, "2018/12/30/13-0.gz"); - - checkNoExist(tmpDir, "2018/12/31/17-0.gz"); - checkNoExist(tmpDir, "2018/12/31/16-0.gz"); - checkNoExist(tmpDir, "2018/12/31/14-0"); - checkNoExist(tmpDir, "2018/12/28/13-0"); - checkNoExist(tmpDir, "2018/12/29/13-0"); - checkNoExist(tmpDir, "2018/12/30/13-0"); - - checkNoExist(tmpDir, "2018/11/20/13-0"); - checkNoExist(tmpDir, "2018/11/20/13-0.gz"); - checkNoExist(tmpDir, "2018/11/21/13-0"); - checkNoExist(tmpDir, "2018/11/21/13-0.gz"); - - makeLogfile(tmpDir, "2018/12/31/16-0", 3); - makeLogfile(tmpDir, "2018/12/31/17-0", 3); - makeLogfile(tmpDir, "2018/12/31/17-1", 1); - makeLogfile(tmpDir, "2018/12/31/17-2", 0); - - dumpFiles(tmpDir, "before second archive maintenance"); + dumpFiles("also before archive maintenance"); + + checkExist("foo/bar"); + checkExist("2018/11/20/13-0"); + checkExist("2018/11/21/13-0"); + checkExist("2018/12/28/13-0"); + checkExist("2018/12/29/13-0"); + checkExist("2018/12/30/13-0"); + checkExist("2018/12/31/14-0"); + checkExist("2018/12/31/16-0"); + checkExist("2018/12/31/17-0"); + checkNoExist("2018/11/20/13-0.gz"); + checkNoExist("2018/11/21/13-0.gz"); + checkNoExist("2018/12/28/13-0.gz"); + checkNoExist("2018/12/29/13-0.gz"); + checkNoExist("2018/12/30/13-0.gz"); + checkNoExist("2018/12/31/14-0.gz"); + checkNoExist("2018/12/31/16-0.gz"); + checkNoExist("2018/12/31/17-0.gz"); + + a.maintenance(); + + dumpFiles("after archive maintenance"); + checkExist("foo/bar"); + checkExist("2018/12/31/17-0"); + checkExist("2018/12/31/16-0"); + checkExist("2018/12/31/14-0.gz"); + checkExist("2018/12/28/13-0.gz"); + checkExist("2018/12/29/13-0.gz"); + checkExist("2018/12/30/13-0.gz"); + + checkNoExist("2018/12/31/17-0.gz"); + checkNoExist("2018/12/31/16-0.gz"); + checkNoExist("2018/12/31/14-0"); + checkNoExist("2018/12/28/13-0"); + checkNoExist("2018/12/29/13-0"); + checkNoExist("2018/12/30/13-0"); + + checkNoExist("2018/11/20/13-0"); + checkNoExist("2018/11/20/13-0.gz"); + checkNoExist("2018/11/21/13-0"); + checkNoExist("2018/11/21/13-0.gz"); + + makeLogfile("2018/12/31/16-0", 3); + makeLogfile("2018/12/31/17-0", 3); + makeLogfile("2018/12/31/17-1", 1); + makeLogfile("2018/12/31/17-2", 0); + + dumpFiles("before second archive maintenance"); a.maintenance(); - dumpFiles(tmpDir, "after second archive maintenance"); + dumpFiles("after second archive maintenance"); - checkExist(tmpDir, "2018/12/31/17-2"); - checkExist(tmpDir, "2018/12/31/17-1"); - checkExist(tmpDir, "2018/12/31/16-0.gz"); - checkExist(tmpDir, "2018/12/31/17-0.gz"); + checkExist("2018/12/31/17-2"); + checkExist("2018/12/31/17-1"); + checkExist("2018/12/31/16-0.gz"); + checkExist("2018/12/31/17-0.gz"); - checkNoExist(tmpDir, "2018/12/31/16-0"); - checkNoExist(tmpDir, "2018/12/31/17-0"); - checkExist(tmpDir, "foo/bar"); + checkNoExist("2018/12/31/16-0"); + checkNoExist("2018/12/31/17-0"); + checkExist("foo/bar"); } - private void dumpFiles(File dir, String header) { + private void dumpFiles(String header) { System.out.println(">>> " + header + " >>> :"); - List seen = scanDir(dir); + List seen = scanDir(tmpDir); seen.sort(null); for (String s : seen) { System.err.println(" " + s); -- cgit v1.2.3