diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-08-12 14:35:50 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2020-08-12 14:35:50 +0000 |
commit | a9f8c9ea4167044634b62cedec8071b5fcb5004d (patch) | |
tree | 47200e3ec877a4fbce3b146cede9caaa3201c619 /logserver/src | |
parent | c2711046304eaa309e6e7d9d69c8835f47fec8fa (diff) |
- Do not do maintenance in the logger thread. That blocks both delivery to the bound Q and holds objects in memory.
Instead just signal that it might be a good idea to do maintenance.
Diffstat (limited to 'logserver/src')
3 files changed, 33 insertions, 43 deletions
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java index 057460323cd..4b586a148a7 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java @@ -2,7 +2,6 @@ package com.yahoo.logserver.handlers; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.logging.Logger; @@ -65,17 +64,7 @@ public class HandlerThread extends Thread implements LogHandler { } private final BlockingQueue<ItemOrList> queue; - private final List<LogHandler> handlers = new ArrayList<LogHandler>(); - private long count; - @SuppressWarnings("unused") - private long droppedCount = 0; - @SuppressWarnings("unused") - private boolean queueWasFull = false; - @SuppressWarnings("unused") - private long lastDropLogMessage = 0; - @SuppressWarnings("unused") - private long lastAcceptingLogMessage = 0; - + private final List<LogHandler> handlers = new ArrayList<>(); public HandlerThread(String name) { super(name); queue = new LinkedBlockingQueue<>(queueSize); @@ -122,25 +111,17 @@ public class HandlerThread extends Thread implements LogHandler { } public void flush() { - Iterator<LogHandler> it = handlers.iterator(); - while (it.hasNext()) { - LogHandler handler = it.next(); + for (LogHandler handler : handlers) { handler.flush(); } } public void close() { - Iterator<LogHandler> it = handlers.iterator(); - while (it.hasNext()) { - LogHandler handler = it.next(); + for (LogHandler handler : handlers) { handler.close(); } } - public long getCount() { - return count; - } - /** * Register a LogHandler */ @@ -191,8 +172,8 @@ public class HandlerThread extends Thread implements LogHandler { throw new NullPointerException("channel is not allowed to be null"); } - // TODO: Make the legmessage elements some kind of composite structure to handle both individual messages and lists uniformly. - List<ItemOrList> drainList = new ArrayList<ItemOrList>(queue.size() + 1); + // TODO: Make the logmessage elements some kind of composite structure to handle both individual messages and lists uniformly. + List<ItemOrList> drainList = new ArrayList<>(queue.size() + 1); try { for (; ; ) { drainList.clear(); @@ -201,10 +182,12 @@ public class HandlerThread extends Thread implements LogHandler { drainList.add(queue.take()); queue.drainTo(drainList); - for (ItemOrList o : drainList) { + for (int i = 0; i < drainList.size(); i++) { // we can get two types of elements here: single log // messages or lists of log messages, so we need to // handle them accordingly. + ItemOrList o = drainList.get(i); + drainList.set(i, o); if (o.item != null) { for (LogHandler handler : handlers) { @@ -217,7 +200,6 @@ public class HandlerThread extends Thread implements LogHandler { } else { throw new IllegalArgumentException("not LogMessage or List: " + o); } - count++; } } } catch (InterruptedException e) { diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java index 43ad5ea7632..520393c3896 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java @@ -7,9 +7,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.zip.GZIPOutputStream; @@ -30,21 +28,28 @@ public class FilesArchived { */ private final File root; + private final Object mutex = new Object(); + // known-existing files inside the archive directory private List<LogFile> knownFiles; - public final static long compressAfterMillis = 2L * 3600 * 1000; - private long maxAgeDays = 30; // GDPR rules: max 30 days - private long sizeLimit = 30L * (1L << 30); // 30 GB + public static final long compressAfterMillis = 2L * 3600 * 1000; + private static final long maxAgeDays = 30; // GDPR rules: max 30 days + private static final long sizeLimit = 30L * (1L << 30); // 30 GB + + private void waitForTrigger(long milliS) throws InterruptedException { + synchronized (mutex) { + mutex.wait(milliS); + } + } private void run() { try { - Thread.sleep(125000); // 2 m 5 s while (true) { while (maintenance()) { - Thread.sleep(2000); // 2 s + waitForTrigger(2000); } - Thread.sleep(299000); // approx 5 min + waitForTrigger(2000); } } catch (Exception e) { // just exit thread on exception, nothing is safe afterwards @@ -78,7 +83,13 @@ public class FilesArchived { return gen; } - public synchronized boolean maintenance() { + public void triggerMaintenance() { + synchronized (mutex) { + mutex.notifyAll(); + } + } + + synchronized boolean maintenance() { boolean action = false; rescan(); if (removeOlderThan(maxAgeDays)) { @@ -130,7 +141,6 @@ public class FilesArchived { // returns true if any files were compressed private boolean compressOldFiles() { - boolean action = false; long now = System.currentTimeMillis(); int count = 0; for (LogFile lf : knownFiles) { @@ -171,7 +181,7 @@ public class FilesArchived { return sum; } - private static Pattern dateFormatRegexp = Pattern.compile(".*/" + + private static final Pattern dateFormatRegexp = Pattern.compile(".*/" + "[0-9][0-9][0-9][0-9]/" + // year "[0-9][0-9]/" + // month "[0-9][0-9]/" + // day @@ -192,9 +202,7 @@ public class FilesArchived { log.warning("skipping file not matching log archive pattern: "+pathName); } } else if (sub.isDirectory()) { - for (LogFile subFile : scanDir(sub)) { - retval.add(subFile); - } + retval.addAll(scanDir(sub)); } } } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java index 9d550a16a7e..e9d0ffc35b8 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java @@ -33,7 +33,7 @@ public class LogWriter { this.archive = archive; this.generation = archive.highestGen(prefix); writer = nextWriter(); - archive.maintenance(); + archive.triggerMaintenance(); } /** @@ -99,7 +99,7 @@ public class LogWriter { public void write(String str) throws IOException { if (writer == null) { writer = nextWriter(); - archive.maintenance(); + archive.triggerMaintenance(); } bytesWritten += str.length(); @@ -110,7 +110,7 @@ public class LogWriter { + currentFile.getAbsolutePath() + "' full, rotating"); writer = nextWriter(); - archive.maintenance(); + archive.triggerMaintenance(); } } |