diff options
author | Arne H Juul <arnej27959@users.noreply.github.com> | 2019-05-22 16:54:18 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-05-22 16:54:18 +0200 |
commit | e36165320f34807bd2ea53e920c5d24c496d44a0 (patch) | |
tree | da9810ce5d828b0c21c5d346525b2656b9e37cd3 /logserver/src/main | |
parent | a15420878d710eb5d782656a0789642195fc00f1 (diff) | |
parent | 8506cbff394e213d9836b4d0f9e0c5f0463f57e7 (diff) |
Merge pull request #9493 from vespa-engine/arnej/add-logarchive-maintainer
add class for managing the log archive
Diffstat (limited to 'logserver/src/main')
4 files changed, 235 insertions, 34 deletions
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java index 6bcd71dfc20..4113cde84dc 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java @@ -45,9 +45,8 @@ public abstract class AbstractLogHandler implements LogHandler { * @param messages List of LogMessage instances. */ public final void handle(List<LogMessage> messages) { - Iterator<LogMessage> it = messages.iterator(); - while (it.hasNext()) { - handle(it.next()); + for (LogMessage l : messages) { + handle(l); } } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index bf7911388dc..6a2940c29c6 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -93,11 +93,12 @@ public class ArchiverHandler extends AbstractLogHandler { */ private LogFilter filter = null; + private FilesArchived filesArchived; + /** - * Creates an ArchiverHandler which puts files under - * the given root directory. + * Creates an ArchiverHandler */ - public ArchiverHandler() { + private ArchiverHandler() { calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); dateformat = new SimpleDateFormat("yyyy/MM/dd/HH"); dateformat.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -137,7 +138,7 @@ public class ArchiverHandler extends AbstractLogHandler { } // invariant: LogWriter we sought was not in the cache - logWriter = new LogWriter(getPrefix(m), maxFileSize); + logWriter = new LogWriter(getPrefix(m), maxFileSize, filesArchived); logWriterLRUCache.put(slot, logWriter); return logWriter; @@ -174,13 +175,7 @@ public class ArchiverHandler extends AbstractLogHandler { */ public String getPrefix(LogMessage msg) { calendar.setTimeInMillis(msg.getTimestamp().toEpochMilli()); -/* - int year = calendar.get(Calendar.YEAR); - int month = calendar.get(Calendar.MONTH) + 1; - int day = calendar.get(Calendar.DAY_OF_MONTH); - int hour = calendar.get(Calendar.HOUR_OF_DAY); -*/ - StringBuffer result = new StringBuffer(absoluteRootDir.length() + StringBuilder result = new StringBuilder(absoluteRootDir.length() + 1 // slash + 4 // year + 1 // slash @@ -189,9 +184,9 @@ public class ArchiverHandler extends AbstractLogHandler { + 2 // day + 1 // slash + 2 // hour - ) - .append(absoluteRootDir).append("/") - .append(dateformat.format(calendar.getTime())); + ); + result.append(absoluteRootDir).append("/") + .append(dateformat.format(calendar.getTime())); return result.toString(); } @@ -244,7 +239,7 @@ public class ArchiverHandler extends AbstractLogHandler { log.log(LogLevel.DEBUG, "Created root at " + absoluteRootDir); } } - + filesArchived = new FilesArchived(root); } public String toString() { diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java new file mode 100644 index 00000000000..fa716921be5 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java @@ -0,0 +1,214 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.logserver.handlers.archive; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.GZIPOutputStream; + + +/** + * This class holds information about all (log) files contained + * in the logarchive directory hierarchy. It also has functionality + * for compressing log files and deleting older files. + * + * @author Arne Juul + */ +public class FilesArchived { + private static final Logger log = Logger.getLogger(FilesArchived.class.getName()); + + /** + * File instance representing root directory of archive + */ + private final File root; + + // known-existing files inside the archive directory + private List<LogFile> knownFiles; + + public final static long compressAfterMillis = 2L * 3600 * 1000; + private long maxAgeDays = 30; // GDPR rules: max 30 days + private long sizeLimit = 30L * (1L << 30); // 30 GB + + /** + * Creates an FilesArchive managing the given directory + */ + public FilesArchived(File rootDir) { + this.root = rootDir; + maintenance(); + } + + public String toString() { + return FilesArchived.class.getName() + ": root=" + root; + } + + public int highestGen(String prefix) { + int gen = 0; + for (LogFile lf : knownFiles) { + if (prefix.equals(lf.prefix)) { + gen = Math.max(gen, lf.generation); + } + } + return gen; + } + + public synchronized void maintenance() { + rescan(); + if (removeOlderThan(maxAgeDays)) rescan(); + if (compressOldFiles()) rescan(); + long days = maxAgeDays; + while (tooMuchDiskUsage() && (--days > 1)) { + if (removeOlderThan(days)) rescan(); + } + } + + private void rescan() { + knownFiles = scanDir(root); + } + + boolean tooMuchDiskUsage() { + long sz = sumFileSizes(); + return sz > sizeLimit; + } + + private boolean olderThan(LogFile lf, long days, long now) { + long mtime = lf.path.lastModified(); + long diff = now - mtime; + return (diff > days * 86400L * 1000L); + } + + // returns true if any files were removed + private boolean removeOlderThan(long days) { + boolean action = false; + long now = System.currentTimeMillis(); + for (LogFile lf : knownFiles) { + if (olderThan(lf, days, now)) { + lf.path.delete(); + log.info("Deleted: "+lf.path); + action = true; + } + } + return action; + } + + // returns true if any files were compressed + private boolean compressOldFiles() { + boolean action = false; + long now = System.currentTimeMillis(); + int count = 0; + for (LogFile lf : knownFiles) { + // avoid compressing entire archive at once + if (lf.canCompress(now) && (count++ < 5)) { + compress(lf.path); + } + } + return count > 0; + } + + private void compress(File oldFile) { + File gzippedFile = new File(oldFile.getPath() + ".gz"); + try { + long mtime = oldFile.lastModified(); + GZIPOutputStream compressor = new GZIPOutputStream(new FileOutputStream(gzippedFile), 0x100000); + FileInputStream inputStream = new FileInputStream(oldFile); + byte [] buffer = new byte[0x100000]; + + for (int read = inputStream.read(buffer); read > 0; read = inputStream.read(buffer)) { + compressor.write(buffer, 0, read); + } + inputStream.close(); + compressor.finish(); + compressor.flush(); + compressor.close(); + oldFile.delete(); + gzippedFile.setLastModified(mtime); + log.info("Compressed: "+gzippedFile); + } catch (IOException e) { + log.warning("Got '" + e + "' while compressing '" + oldFile.getPath() + "'."); + } + } + + public long sumFileSizes() { + long sum = 0; + for (LogFile lf : knownFiles) { + sum += lf.path.length(); + } + return sum; + } + + private static List<LogFile> scanDir(File top) { + List<LogFile> retval = new ArrayList<>(); + String[] names = top.list(); + if (names != null) { + for (String name : names) { + File sub = new File(top, name); + if (sub.isFile()) { + retval.add(new LogFile(sub)); + } else if (sub.isDirectory()) { + for (LogFile subFile : scanDir(sub)) { + retval.add(subFile); + } + } + } + } + return retval; + } + + static class LogFile { + public final File path; + public final String prefix; + public final int generation; + public final boolean zsuff; + + public boolean canCompress(long now) { + if (zsuff) return false; // already compressed + if (! path.isFile()) return false; // not a file + long diff = now - path.lastModified(); + if (diff < compressAfterMillis) return false; // too new + return true; + } + + private static int generationOf(String name) { + int dash = name.lastIndexOf('-'); + if (dash < 0) return 0; + String suff = name.substring(dash + 1); + int r = 0; + for (char ch : suff.toCharArray()) { + if (ch >= '0' && ch <= '9') { + r *= 10; + r += (ch - '0'); + } else { + break; + } + } + return r; + } + private static String prefixOf(String name) { + int dash = name.lastIndexOf('-'); + if (dash < 0) return name; + return name.substring(0, dash); + } + private static boolean zSuffix(String name) { + if (name.endsWith(".gz")) return true; + // add other compression suffixes here + return false; + } + public LogFile(File path) { + String name = path.toString(); + this.path = path; + this.prefix = prefixOf(name); + this.generation = generationOf(name); + this.zsuff = zSuffix(name); + } + public String toString() { + return "FilesArchived.LogFile{name="+path+" prefix="+prefix+" gen="+generation+" z="+zsuff+"}"; + } + } +} + diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java index 8d0eeb004fb..47a9b04291d 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java @@ -14,22 +14,26 @@ import com.yahoo.log.LogLevel; * * @author Bjorn Borud */ -public class LogWriter extends Writer { +public class LogWriter { private static final Logger log = Logger.getLogger(LogWriter.class.getName()); private long bytesWritten = 0; - private int generation = 0; + private int generation; private int maxSize = 20 * (1024 * 1024); private final int resumeLimit = 95; private final int resumeLimitSize = (maxSize * resumeLimit / 100); private File currentFile; private Writer writer; private final String prefix; + private final FilesArchived archive; - public LogWriter(String prefix, int maxSize) throws IOException { + public LogWriter(String prefix, int maxSize, FilesArchived archive) throws IOException { this.prefix = prefix; this.maxSize = maxSize; + this.archive = archive; + this.generation = archive.highestGen(prefix); writer = nextWriter(); + archive.maintenance(); } /** @@ -96,22 +100,10 @@ public class LogWriter extends Writer { throw new RuntimeException("Unable to create next log file"); } - /** - * Note that this method should not be used directly since - * that would circumvent rotation when it grows past its - * maximum size. use the one that takes String instead. - * <p> - * <em> - * (This is a class which is only used internally anyway) - * </em> - */ - public void write(char[] cbuff, int offset, int len) throws IOException { - throw new RuntimeException("This method should not be used"); - } - public void write(String str) throws IOException { if (writer == null) { writer = nextWriter(); + archive.maintenance(); } bytesWritten += str.length(); @@ -122,6 +114,7 @@ public class LogWriter extends Writer { + currentFile.getAbsolutePath() + "' full, rotating"); writer = nextWriter(); + archive.maintenance(); } } |