- Do not do maintenance in the logger thread. That blocks both delivery to the bound Q and holds objects in memory.

  Instead just signal that it might be a good idea to do maintenance.

3 files changed, 33 insertions, 43 deletions

diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java
index 057460323cd..4b586a148a7 100644
--- a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java
@@ -2,7 +2,6 @@
 package com.yahoo.logserver.handlers;
 
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
 import java.util.logging.Logger;
 
@@ -65,17 +64,7 @@ public class HandlerThread extends Thread implements LogHandler {
     }
 
     private final BlockingQueue<ItemOrList> queue;
-    private final List<LogHandler> handlers = new ArrayList<LogHandler>();
-    private long count;
-    @SuppressWarnings("unused")
-    private long droppedCount = 0;
-    @SuppressWarnings("unused")
-    private boolean queueWasFull = false;
-    @SuppressWarnings("unused")
-    private long lastDropLogMessage = 0;
-    @SuppressWarnings("unused")
-    private long lastAcceptingLogMessage = 0;
-
+    private final List<LogHandler> handlers = new ArrayList<>();
     public HandlerThread(String name) {
         super(name);
         queue = new LinkedBlockingQueue<>(queueSize);
@@ -122,25 +111,17 @@ public class HandlerThread extends Thread implements LogHandler {
     }
 
     public void flush() {
-        Iterator<LogHandler> it = handlers.iterator();
-        while (it.hasNext()) {
-            LogHandler handler = it.next();
+        for (LogHandler handler : handlers) {
             handler.flush();
         }
     }
 
     public void close() {
-        Iterator<LogHandler> it = handlers.iterator();
-        while (it.hasNext()) {
-            LogHandler handler = it.next();
+        for (LogHandler handler : handlers) {
             handler.close();
         }
     }
 
-    public long getCount() {
-        return count;
-    }
-
     /**
      * Register a LogHandler
      */
@@ -191,8 +172,8 @@ public class HandlerThread extends Thread implements LogHandler {
             throw new NullPointerException("channel is not allowed to be null");
         }
 
-        // TODO: Make the legmessage elements some kind of composite structure to handle both individual messages and lists uniformly.
-        List<ItemOrList> drainList = new ArrayList<ItemOrList>(queue.size() + 1);
+        // TODO: Make the logmessage elements some kind of composite structure to handle both individual messages and lists uniformly.
+        List<ItemOrList> drainList = new ArrayList<>(queue.size() + 1);
         try {
             for (; ; ) {
                 drainList.clear();
@@ -201,10 +182,12 @@ public class HandlerThread extends Thread implements LogHandler {
                 drainList.add(queue.take());
                 queue.drainTo(drainList);
 
-                for (ItemOrList o : drainList) {
+                for (int i = 0; i < drainList.size(); i++) {
                     // we can get two types of elements here: single log
                     // messages or lists of log messages, so we need to
                     // handle them accordingly.
+                    ItemOrList o = drainList.get(i);
+                    drainList.set(i, o);
 
                     if (o.item != null) {
                         for (LogHandler handler : handlers) {
@@ -217,7 +200,6 @@ public class HandlerThread extends Thread implements LogHandler {
                     } else {
                         throw new IllegalArgumentException("not LogMessage or List: " + o);
                     }
-                    count++;
                 }
             }
         } catch (InterruptedException e) {
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
index 43ad5ea7632..520393c3896 100644
--- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
@@ -7,9 +7,7 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPOutputStream;
@@ -30,21 +28,28 @@ public class FilesArchived {
      */
     private final File root;
 
+    private final Object mutex = new Object();
+
     // known-existing files inside the archive directory
     private List<LogFile> knownFiles;
 
-    public final static long compressAfterMillis = 2L * 3600 * 1000;
-    private long maxAgeDays = 30; // GDPR rules: max 30 days
-    private long sizeLimit = 30L * (1L << 30); // 30 GB
+    public static final long compressAfterMillis = 2L * 3600 * 1000;
+    private static final long maxAgeDays = 30; // GDPR rules: max 30 days
+    private static final long sizeLimit = 30L * (1L << 30); // 30 GB
+
+    private void waitForTrigger(long milliS) throws InterruptedException {
+        synchronized (mutex) {
+            mutex.wait(milliS);
+        }
+    }
 
     private void run() {
         try {
-            Thread.sleep(125000); // 2 m 5 s
             while (true) {
                 while (maintenance()) {
-                    Thread.sleep(2000); // 2 s
+                    waitForTrigger(2000);
                 }
-                Thread.sleep(299000); // approx 5 min
+                waitForTrigger(2000);
             }
         } catch (Exception e) {
             // just exit thread on exception, nothing is safe afterwards
@@ -78,7 +83,13 @@ public class FilesArchived {
         return gen;
     }
 
-    public synchronized boolean maintenance() {
+    public void triggerMaintenance() {
+        synchronized (mutex) {
+            mutex.notifyAll();
+        }
+    }
+
+    synchronized boolean maintenance() {
         boolean action = false;
         rescan();
         if (removeOlderThan(maxAgeDays)) {
@@ -130,7 +141,6 @@ public class FilesArchived {
 
     // returns true if any files were compressed
     private boolean compressOldFiles() {
-        boolean action = false;
         long now = System.currentTimeMillis();
         int count = 0;
         for (LogFile lf : knownFiles) {
@@ -171,7 +181,7 @@ public class FilesArchived {
         return sum;
     }
 
-    private static Pattern dateFormatRegexp = Pattern.compile(".*/" +
+    private static final Pattern dateFormatRegexp = Pattern.compile(".*/" +
             "[0-9][0-9][0-9][0-9]/" + // year
             "[0-9][0-9]/" + // month
             "[0-9][0-9]/" + // day
@@ -192,9 +202,7 @@ public class FilesArchived {
                         log.warning("skipping file not matching log archive pattern: "+pathName);
                     }
                 } else if (sub.isDirectory()) {
-                    for (LogFile subFile : scanDir(sub)) {
-                        retval.add(subFile);
-                    }
+                    retval.addAll(scanDir(sub));
                 }
             }
         }
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java
index 9d550a16a7e..e9d0ffc35b8 100644
--- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java
@@ -33,7 +33,7 @@ public class LogWriter {
         this.archive = archive;
         this.generation = archive.highestGen(prefix);
         writer = nextWriter();
-        archive.maintenance();
+        archive.triggerMaintenance();
     }
 
     /**
@@ -99,7 +99,7 @@ public class LogWriter {
     public void write(String str) throws IOException {
         if (writer == null) {
             writer = nextWriter();
-            archive.maintenance();
+            archive.triggerMaintenance();
         }
 
         bytesWritten += str.length();
@@ -110,7 +110,7 @@ public class LogWriter {
                              + currentFile.getAbsolutePath()
                              + "' full, rotating");
             writer = nextWriter();
-            archive.maintenance();
+            archive.triggerMaintenance();
         }
     }