Merge pull request #9493 from vespa-engine/arnej/add-logarchive-maintainer

add class for managing the log archive

6 files changed, 364 insertions, 35 deletions

diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java
index 6bcd71dfc20..4113cde84dc 100644
--- a/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java
@@ -45,9 +45,8 @@ public abstract class AbstractLogHandler implements LogHandler {
      * @param messages List of LogMessage instances.
      */
     public final void handle(List<LogMessage> messages) {
-        Iterator<LogMessage> it = messages.iterator();
-        while (it.hasNext()) {
-            handle(it.next());
+        for (LogMessage l : messages) {
+            handle(l);
         }
     }
 
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java
index bf7911388dc..6a2940c29c6 100644
--- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java
@@ -93,11 +93,12 @@ public class ArchiverHandler extends AbstractLogHandler {
      */
     private LogFilter filter = null;
 
+    private FilesArchived filesArchived;
+
     /**
-     * Creates an ArchiverHandler which puts files under
-     * the given root directory.
+     * Creates an ArchiverHandler
      */
-    public ArchiverHandler() {
+    private ArchiverHandler() {
         calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
         dateformat = new SimpleDateFormat("yyyy/MM/dd/HH");
         dateformat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -137,7 +138,7 @@ public class ArchiverHandler extends AbstractLogHandler {
         }
 
         // invariant: LogWriter we sought was not in the cache
-        logWriter = new LogWriter(getPrefix(m), maxFileSize);
+        logWriter = new LogWriter(getPrefix(m), maxFileSize, filesArchived);
         logWriterLRUCache.put(slot, logWriter);
 
         return logWriter;
@@ -174,13 +175,7 @@ public class ArchiverHandler extends AbstractLogHandler {
      */
     public String getPrefix(LogMessage msg) {
         calendar.setTimeInMillis(msg.getTimestamp().toEpochMilli());
-/*
-        int year   = calendar.get(Calendar.YEAR);
-        int month  = calendar.get(Calendar.MONTH) + 1;
-        int day    = calendar.get(Calendar.DAY_OF_MONTH);
-        int hour   = calendar.get(Calendar.HOUR_OF_DAY);
-*/
-        StringBuffer result = new StringBuffer(absoluteRootDir.length()
+        StringBuilder result = new StringBuilder(absoluteRootDir.length()
                                                        + 1 // slash
                                                        + 4 // year
                                                        + 1 // slash
@@ -189,9 +184,9 @@ public class ArchiverHandler extends AbstractLogHandler {
                                                        + 2 // day
                                                        + 1 // slash
                                                        + 2 // hour
-        )
-                .append(absoluteRootDir).append("/")
-                .append(dateformat.format(calendar.getTime()));
+        );
+        result.append(absoluteRootDir).append("/")
+              .append(dateformat.format(calendar.getTime()));
         return result.toString();
     }
 
@@ -244,7 +239,7 @@ public class ArchiverHandler extends AbstractLogHandler {
                 log.log(LogLevel.DEBUG, "Created root at " + absoluteRootDir);
             }
         }
-
+        filesArchived = new FilesArchived(root);
     }
 
     public String toString() {
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
new file mode 100644
index 00000000000..fa716921be5
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
@@ -0,0 +1,214 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.logserver.handlers.archive;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.GZIPOutputStream;
+
+
+/**
+ * This class holds information about all (log) files contained
+ * in the logarchive directory hierarchy.  It also has functionality
+ * for compressing log files and deleting older files.
+ *
+ * @author Arne Juul
+ */
+public class FilesArchived {
+    private static final Logger log = Logger.getLogger(FilesArchived.class.getName());
+
+    /**
+     * File instance representing root directory of archive
+     */
+    private final File root;
+
+    // known-existing files inside the archive directory
+    private List<LogFile> knownFiles;
+
+    public final static long compressAfterMillis = 2L * 3600 * 1000;
+    private long maxAgeDays = 30; // GDPR rules: max 30 days
+    private long sizeLimit = 30L * (1L << 30); // 30 GB
+
+    /**
+     * Creates an FilesArchive managing the given directory
+     */
+    public FilesArchived(File rootDir) {
+        this.root = rootDir;
+        maintenance();
+    }
+
+    public String toString() {
+        return FilesArchived.class.getName() + ": root=" + root;
+    }
+
+    public int highestGen(String prefix) {
+        int gen = 0;
+        for (LogFile lf : knownFiles) {
+            if (prefix.equals(lf.prefix)) {
+                gen = Math.max(gen, lf.generation);
+            }
+        }
+        return gen;
+    }
+
+    public synchronized void maintenance() {
+        rescan();
+        if (removeOlderThan(maxAgeDays)) rescan();
+        if (compressOldFiles()) rescan();
+        long days = maxAgeDays;
+        while (tooMuchDiskUsage() && (--days > 1)) {
+            if (removeOlderThan(days)) rescan();
+        }
+    }
+
+    private void rescan() {
+        knownFiles = scanDir(root);
+    }
+
+    boolean tooMuchDiskUsage() {
+        long sz = sumFileSizes();
+        return sz > sizeLimit;
+    }
+
+    private boolean olderThan(LogFile lf, long days, long now) {
+        long mtime = lf.path.lastModified();
+        long diff = now - mtime;
+        return (diff > days * 86400L * 1000L);
+    }
+
+    // returns true if any files were removed
+    private boolean removeOlderThan(long days) {
+        boolean action = false;
+        long now = System.currentTimeMillis();
+        for (LogFile lf : knownFiles) {
+            if (olderThan(lf, days, now)) {
+                lf.path.delete();
+                log.info("Deleted: "+lf.path);
+                action = true;
+            }
+        }
+        return action;
+    }
+
+    // returns true if any files were compressed
+    private boolean compressOldFiles() {
+        boolean action = false;
+        long now = System.currentTimeMillis();
+        int count = 0;
+        for (LogFile lf : knownFiles) {
+            // avoid compressing entire archive at once
+            if (lf.canCompress(now) && (count++ < 5)) {
+                compress(lf.path);
+            }
+        }
+        return count > 0;
+    }
+
+    private void compress(File oldFile) {
+        File gzippedFile = new File(oldFile.getPath() + ".gz");
+        try {
+            long mtime = oldFile.lastModified();
+            GZIPOutputStream compressor = new GZIPOutputStream(new FileOutputStream(gzippedFile), 0x100000);
+            FileInputStream inputStream = new FileInputStream(oldFile);
+            byte [] buffer = new byte[0x100000];
+
+            for (int read = inputStream.read(buffer); read > 0; read = inputStream.read(buffer)) {
+                compressor.write(buffer, 0, read);
+            }
+            inputStream.close();
+            compressor.finish();
+            compressor.flush();
+            compressor.close();
+            oldFile.delete();
+            gzippedFile.setLastModified(mtime);
+            log.info("Compressed: "+gzippedFile);
+        } catch (IOException e) {
+            log.warning("Got '" + e + "' while compressing '" + oldFile.getPath() + "'.");
+        }
+    }
+
+    public long sumFileSizes() {
+        long sum = 0;
+        for (LogFile lf : knownFiles) {
+            sum += lf.path.length();
+        }
+        return sum;
+    }
+
+    private static List<LogFile> scanDir(File top) {
+        List<LogFile> retval = new ArrayList<>();
+        String[] names = top.list();
+        if (names != null) {
+            for (String name : names) {
+                File sub = new File(top, name);
+                if (sub.isFile()) {
+                    retval.add(new LogFile(sub));
+                } else if (sub.isDirectory()) {
+                    for (LogFile subFile : scanDir(sub)) {
+                        retval.add(subFile);
+                    }
+                }
+            }
+        }
+        return retval;
+    }
+
+    static class LogFile {
+        public final File path; 
+        public final String prefix;
+        public final int generation;
+        public final boolean zsuff;
+
+        public boolean canCompress(long now) {
+            if (zsuff) return false; // already compressed
+            if (! path.isFile()) return false; // not a file
+            long diff = now - path.lastModified();
+            if (diff < compressAfterMillis) return false; // too new
+            return true;
+        }
+
+        private static int generationOf(String name) {
+            int dash = name.lastIndexOf('-');
+            if (dash < 0) return 0;
+            String suff = name.substring(dash + 1);
+            int r = 0;
+            for (char ch : suff.toCharArray()) {
+                if (ch >= '0' && ch <= '9') {
+                    r *= 10;
+                    r += (ch - '0');
+                } else {
+                    break;
+                }
+            }
+            return r;
+        }
+        private static String prefixOf(String name) {
+            int dash = name.lastIndexOf('-');
+            if (dash < 0) return name;
+            return name.substring(0, dash);
+        }
+        private static boolean zSuffix(String name) {
+            if (name.endsWith(".gz")) return true;
+            // add other compression suffixes here
+            return false;
+        }
+        public LogFile(File path) {
+            String name = path.toString();
+            this.path = path;
+            this.prefix = prefixOf(name);
+            this.generation = generationOf(name);
+            this.zsuff = zSuffix(name);
+        }
+        public String toString() {
+            return "FilesArchived.LogFile{name="+path+" prefix="+prefix+" gen="+generation+" z="+zsuff+"}";
+        }
+    }
+}
+
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java
index 8d0eeb004fb..47a9b04291d 100644
--- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java
@@ -14,22 +14,26 @@ import com.yahoo.log.LogLevel;
  *
  * @author Bjorn Borud
  */
-public class LogWriter extends Writer {
+public class LogWriter {
     private static final Logger log = Logger.getLogger(LogWriter.class.getName());
 
     private long bytesWritten = 0;
-    private int generation = 0;
+    private int generation;
     private int maxSize = 20 * (1024 * 1024);
     private final int resumeLimit = 95;
     private final int resumeLimitSize = (maxSize * resumeLimit / 100);
     private File currentFile;
     private Writer writer;
     private final String prefix;
+    private final FilesArchived archive;
 
-    public LogWriter(String prefix, int maxSize) throws IOException {
+    public LogWriter(String prefix, int maxSize, FilesArchived archive) throws IOException {
         this.prefix = prefix;
         this.maxSize = maxSize;
+        this.archive = archive;
+        this.generation = archive.highestGen(prefix);
         writer = nextWriter();
+        archive.maintenance();
     }
 
     /**
@@ -96,22 +100,10 @@ public class LogWriter extends Writer {
         throw new RuntimeException("Unable to create next log file");
     }
 
-    /**
-     * Note that this method should not be used directly since
-     * that would circumvent rotation when it grows past its
-     * maximum size.  use the one that takes String instead.
-     * <p>
-     * <em>
-     * (This is a class which is only used internally anyway)
-     * </em>
-     */
-    public void write(char[] cbuff, int offset, int len) throws IOException {
-        throw new RuntimeException("This method should not be used");
-    }
-
     public void write(String str) throws IOException {
         if (writer == null) {
             writer = nextWriter();
+            archive.maintenance();
         }
 
         bytesWritten += str.length();
@@ -122,6 +114,7 @@ public class LogWriter extends Writer {
                              + currentFile.getAbsolutePath()
                              + "' full, rotating");
             writer = nextWriter();
+            archive.maintenance();
         }
     }
 
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java
index 578caab366c..6fe8b3f0436 100644
--- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java
@@ -213,8 +213,10 @@ public class ArchiverHandlerTestCase {
     @Test
     public void testCacheEldestEntry() throws IOException {
         LogWriterLRUCache cache = new LogWriterLRUCache(5, (float) 0.75);
+        String d = "target/tmp/logarchive";
+        FilesArchived archive = new FilesArchived(new File(d));
         for (int i = 0; i < cache.maxEntries + 10; i++) {
-            cache.put(i, new LogWriter("/tmp/", 5));
+            cache.put(i, new LogWriter(d+"/2018/12/31/17", 5, archive));
         }
         assertEquals(cache.size(), cache.maxEntries);
     }
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java
new file mode 100644
index 00000000000..c4f60e36d64
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java
@@ -0,0 +1,126 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.logserver.handlers.archive;
+
+import com.yahoo.io.IOUtils;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author Arne Juul
+ */
+public class FilesArchivedTestCase {
+
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    private void makeLogfile(File dir, String name, long hours) throws IOException {
+        File f = new File(dir, name);
+        f.getParentFile().mkdirs();
+        new FileWriter(f).write("foo bar baz\n");
+        long now = System.currentTimeMillis();
+        f.setLastModified(now - (hours * 3600 * 1000));
+    }
+
+    void checkExist(File dir, String name) {
+        assertTrue(new File(dir, name).isFile());
+    }
+    void checkNoExist(File dir, String name) {
+        assertFalse(new File(dir, name).isFile());
+    }
+
+    @Test
+    public void testMaintenance() throws java.io.IOException {
+        File tmpDir = temporaryFolder.newFolder();
+        try {
+            makeLogfile(tmpDir, "2018/11/20/13-0", 35*24);
+            makeLogfile(tmpDir, "2018/11/21/13-0", 34*24);
+            makeLogfile(tmpDir, "2018/12/28/13-0", 3*24);
+            makeLogfile(tmpDir, "2018/12/29/13-0", 2*24);
+            makeLogfile(tmpDir, "2018/12/30/13-0", 1*24);
+            makeLogfile(tmpDir, "2018/12/31/14-0", 3);
+            makeLogfile(tmpDir, "2018/12/31/16-0", 1);
+            makeLogfile(tmpDir, "2018/12/31/17-0", 0);
+            dumpFiles(tmpDir, "before archive maintenance");
+            FilesArchived a = new FilesArchived(tmpDir);
+            dumpFiles(tmpDir, "after archive maintenance");
+            checkExist(tmpDir, "2018/12/31/17-0");
+            checkExist(tmpDir, "2018/12/31/16-0");
+            checkExist(tmpDir, "2018/12/31/14-0.gz");
+            checkExist(tmpDir, "2018/12/28/13-0.gz");
+            checkExist(tmpDir, "2018/12/29/13-0.gz");
+            checkExist(tmpDir, "2018/12/30/13-0.gz");
+
+            checkNoExist(tmpDir, "2018/12/31/17-0.gz");
+            checkNoExist(tmpDir, "2018/12/31/16-0.gz");
+            checkNoExist(tmpDir, "2018/12/31/14-0");
+            checkNoExist(tmpDir, "2018/12/28/13-0");
+            checkNoExist(tmpDir, "2018/12/29/13-0");
+            checkNoExist(tmpDir, "2018/12/30/13-0");
+
+            checkNoExist(tmpDir, "2018/11/20/13-0");
+            checkNoExist(tmpDir, "2018/11/20/13-0.gz");
+            checkNoExist(tmpDir, "2018/11/21/13-0");
+            checkNoExist(tmpDir, "2018/11/21/13-0.gz");
+
+            makeLogfile(tmpDir, "2018/12/31/16-0", 3);
+            makeLogfile(tmpDir, "2018/12/31/17-0", 3);
+            makeLogfile(tmpDir, "2018/12/31/17-1", 1);
+            makeLogfile(tmpDir, "2018/12/31/17-2", 0);
+
+            dumpFiles(tmpDir, "before second archive maintenance");
+            a.maintenance();
+            dumpFiles(tmpDir, "after second archive maintenance");
+
+            checkExist(tmpDir, "2018/12/31/17-2");
+            checkExist(tmpDir, "2018/12/31/17-1");
+            checkExist(tmpDir, "2018/12/31/16-0.gz");
+            checkExist(tmpDir, "2018/12/31/17-0.gz");
+
+            checkNoExist(tmpDir, "2018/12/31/16-0");
+            checkNoExist(tmpDir, "2018/12/31/17-0");
+        } finally {
+            IOUtils.recursiveDeleteDir(tmpDir);
+        }
+    }
+
+    private void dumpFiles(File dir, String header) {
+        System.out.println(">>> " + header + " >>> :");
+        List<String> seen = scanDir(dir);
+        seen.sort(null);
+        for (String s : seen) {
+            System.err.println("   " + s);
+        }
+        System.out.println("<<< " + header + " <<<");
+    }
+
+    private static List<String> scanDir(File top) {
+        List<String> retval = new ArrayList<>();
+        String[] names = top.list();
+        if (names != null) {
+            for (String name : names) {
+                File sub = new File(top, name);
+                if (sub.isFile()) {
+                    retval.add(sub.toString());
+                } else if (sub.isDirectory()) {
+                    for (String subFile : scanDir(sub)) {
+                        retval.add(subFile);
+                    }
+                }
+            }
+        }
+        return retval;
+    }
+
+}