Add missing logserver files

4 files changed, 520 insertions, 0 deletions

diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java
new file mode 100644
index 00000000000..06bf5183005
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java
@@ -0,0 +1,255 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.archive;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+import java.util.Iterator;
+import java.util.TimeZone;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.yahoo.logserver.filter.LogFilter;
+import com.yahoo.logserver.filter.LogFilterManager;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.handlers.AbstractLogHandler;
+
+
+/**
+ * This class implements a log handler which archives the incoming
+ * messages based on their timestamp.  The goal of this archiver
+ * is to make it easy to locate messages in a time interval, while
+ * ensuring that no log file exceeds the maximum allowed size.
+ *
+ * <p>
+ * This class is not thread safe.
+ * </p>
+ *
+ * <p>
+ * TODO:
+ * </p>
+ * <ul>
+ *  <li> Add file locking support in order to make it
+ *       possible to do concurrent compression of log
+ *       files.
+ *
+ *  <li> Add support for disk monitoring.  Should have
+ *       high/low watermark mechanism and three modes
+ *       of operation: normal, tight and full.  In
+ *       "tight" mode disk is running low and compression
+ *       and cleanup should possibly be more frequent.
+ *
+ *  <li> Add compression task which periodically scans
+ *       the log directory looking for uncompressed
+ *       candidate log files.
+ *
+ * </ul>
+ * @author  <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class ArchiverHandler extends AbstractLogHandler
+{
+    private static final Logger log
+        = Logger.getLogger(ArchiverHandler.class.getName());
+
+    /** File instance representing root directory for logging */
+    private File root;
+
+    /** Root directory for logging */
+    private String absoluteRootDir;
+
+    /** Max number of log files open at any given time */
+    private final int maxFilesOpen = 100;
+
+    /**
+     *The maximum number of bytes we allow a file to grow to
+     * before we rotate it
+     */
+    private int maxFileSize;
+
+    /** Calendar instance for operating on Date objects */
+    private final Calendar calendar;
+
+    /** DateFormat instance for building filenames */
+    private final SimpleDateFormat dateformat;
+
+    /**
+     * This is an LRU cache for LogWriter objects.  Remember that
+     * we have one LogWriter for each time slot
+     */
+    private final LogWriterLRUCache logWriterLRUCache;
+
+    /**
+     * Filtering
+     */
+    private LogFilter filter = null;
+
+    /**
+     * Creates an ArchiverHandler which puts files under
+     * the given root directory.
+     */
+    public ArchiverHandler () {
+        calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
+        dateformat = new SimpleDateFormat("yyyy/MM/dd/HH");
+        dateformat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+        // Set up filtering
+        String archiveMetrics = System.getProperty("vespa_log_server__archive_metric");
+        if ("off".equals(archiveMetrics)){
+            filter = LogFilterManager.getLogFilter("system.nometricsevents");
+        }
+
+        setLogFilter(filter);
+
+        // set up LRU for files
+        logWriterLRUCache = new LogWriterLRUCache(maxFilesOpen,
+                                                  (float)0.75);
+    }
+
+    /**
+     * Creates an ArchiverHandler which puts files under
+     * the given root directory.
+     */
+    public ArchiverHandler (String rootDir, int maxFileSize) {
+        this();
+        setRootDir(rootDir);
+        this.maxFileSize = maxFileSize;
+    }
+
+
+    /**
+     * Return the appropriate LogWriter given a log message.
+     */
+    private synchronized LogWriter getLogWriter (LogMessage m) throws IOException {
+        Integer slot = new Integer(dateHash(m.getTime()));
+        LogWriter logWriter = logWriterLRUCache.get(slot);
+        if (logWriter != null) {
+            return logWriter;
+        }
+
+        // invariant: LogWriter we sought was not in the cache
+        logWriter = new LogWriter(getPrefix(m), maxFileSize);
+        logWriterLRUCache.put(slot, logWriter);
+
+        return logWriter;
+    }
+
+    /**
+     * This method is just a fancy way of generating a stripped
+     * down number representing year, month, day and hour in order
+     * to partition logging in time.
+     *
+     * <P>
+     * This method is not thread-safe.
+     */
+    public int dateHash (long time) {
+        calendar.setTimeInMillis(time);
+        int year   = calendar.get(Calendar.YEAR);
+        int month  = calendar.get(Calendar.MONTH) + 1;
+        int day    = calendar.get(Calendar.DAY_OF_MONTH);
+        int hour   = calendar.get(Calendar.HOUR_OF_DAY);
+
+        return year * 1000000
+            + month * 10000
+            + day   * 100
+            + hour;
+    }
+
+    /**
+     * Generate prefix for log filenames based on log message.
+     *
+     * <P>
+     * <EM>This message is <code>public</code> only because we need
+     * access to it in unit tests.  For all practical purposes this
+     * method does not exist to you, the application programmer, OK? :-)</EM>
+     * <P>
+     * XXX optimize!
+     */
+    public String getPrefix (LogMessage msg) {
+        calendar.setTimeInMillis(msg.getTime());
+/*
+        int year   = calendar.get(Calendar.YEAR);
+        int month  = calendar.get(Calendar.MONTH) + 1;
+        int day    = calendar.get(Calendar.DAY_OF_MONTH);
+        int hour   = calendar.get(Calendar.HOUR_OF_DAY);
+*/
+        StringBuffer result = new StringBuffer(absoluteRootDir.length()
+                + 1 // slash
+                + 4 // year
+                + 1 // slash
+                + 2 // month
+                + 1 // slash
+                + 2 // day
+                + 1 // slash
+                + 2 // hour
+        )
+                .append(absoluteRootDir).append("/")
+                .append(dateformat.format(calendar.getTime()));
+        return result.toString();
+    }
+
+    public boolean doHandle (LogMessage msg) {
+        try {
+            LogWriter logWriter = getLogWriter(msg);
+            logWriter.write(msg.toString());
+        }
+        catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return true;
+    }
+
+    public synchronized void flush () {
+        for (LogWriter l : logWriterLRUCache.values()) {
+            try {
+                l.flush();
+            }
+            catch (IOException e) {
+                log.log(Level.WARNING, "Flushing failed", e);
+            }
+        }
+    }
+
+    public synchronized void close () {
+        Iterator<LogWriter> it = logWriterLRUCache.values().iterator();
+        while (it.hasNext()) {
+            LogWriter l = it.next();
+            try {
+                l.close();
+            }
+            catch (IOException e) {
+                log.log(Level.WARNING, "Closing failed", e);
+            }
+            it.remove();
+        }
+    }
+
+    private void setRootDir(String rootDir) {
+        // roundabout way of setting things, but this way we can
+        // get around Java's ineptitude for file handling. (relative
+        // paths in File are broken)
+        //
+        absoluteRootDir = new File(rootDir).getAbsolutePath();
+        root = new File(absoluteRootDir);
+
+        // ensure that root dir exists
+        if (root.isDirectory()) {
+            log.log(LogLevel.DEBUG, "Using " + absoluteRootDir + " as root");
+        } else {
+            if (! root.mkdirs()) {
+                log.log(LogLevel.ERROR,
+                        "Unable to create directory " + absoluteRootDir);
+            } else {
+                log.log(LogLevel.DEBUG, "Created root at " + absoluteRootDir);
+            }
+        }
+
+    }
+
+    public String toString () {
+        return ArchiverHandler.class.getName() + ": root=" + absoluteRootDir;
+    }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java
new file mode 100644
index 00000000000..0d04b4cf981
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java
@@ -0,0 +1,77 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.archive;
+
+import java.util.logging.Logger;
+import com.yahoo.logserver.Server;
+import com.yahoo.plugin.Config;
+import com.yahoo.plugin.Plugin;
+
+
+public class ArchiverPlugin implements Plugin
+{
+    /**
+     * Default log archive dir (relative to current directory
+     * at startup).
+     */
+    private static final String DEFAULT_DIR = "logarchive";
+
+    /**
+     * Default max file size for archived log files.
+     */
+    private static final String DEFAULT_MAXFILESIZE = "20971520";
+
+    private final Server server = Server.getInstance();
+    private static final Logger log =
+        Logger.getLogger(ArchiverPlugin.class.getName());
+    private ArchiverHandler archiver;
+
+    /**
+     * @return the name of this plugin
+     */
+    public String getPluginName() {
+        return "logarchive";
+    }
+
+    /** Initialize the archiver plugin
+     *
+     * Config keys used:
+     *
+     *   maxfilesize
+     *   dir            The root of the logarchive, make sure this does
+     *                  <b>not</b> end with a '/' character.
+     */
+    public void initPlugin(Config config) {
+
+        if (archiver != null) {
+        	log.finer("ArchivePlugin doubly initialized");
+            throw new IllegalStateException("plugin already initialized: "
+                                            + getPluginName());
+        }
+
+        // Possible to disable logarchive for testing
+        String rootDir = config.get("dir", DEFAULT_DIR);
+        int maxFileSize = config.getInt("maxfilesize", DEFAULT_MAXFILESIZE);
+        String threadName = config.get("thread", getPluginName());
+
+        // register log handler and flusher
+        archiver = new ArchiverHandler(rootDir, maxFileSize);
+        server.registerLogHandler(archiver, threadName);
+        server.registerFlusher(archiver);
+    }
+
+    /**
+     * Shut down the archiver plugin.
+     */
+    public void shutdownPlugin() {
+
+        if (archiver == null) {
+        	log.finer("ArchiverPlugin shutdown before initialize");
+            throw new IllegalStateException("plugin not initialized: "
+                                            + getPluginName());
+        }
+        server.unregisterLogHandler(archiver);
+        server.unregisterFlusher(archiver);
+        archiver.close();
+        archiver = null;
+    }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java
new file mode 100644
index 00000000000..fb78e8243c3
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java
@@ -0,0 +1,147 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.archive;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.logging.Logger;
+
+import com.yahoo.log.LogLevel;
+
+/**
+ * This class is not thread-safe.
+ *
+ *
+ * @author  <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class LogWriter extends Writer
+{
+    private static final Logger log = Logger.getLogger(LogWriter.class.getName());
+
+    private long bytesWritten      = 0;
+    private int generation         = 0;
+    private int maxSize            = 20 * (1024*1024);
+    private final int resumeLimit        = 95;
+    private final int resumeLimitSize    = (maxSize * resumeLimit / 100);
+    private File currentFile;
+    private Writer writer;
+    private final String prefix;
+
+    public LogWriter (String prefix, int maxSize) throws IOException {
+        this.prefix = prefix;
+        this.maxSize = maxSize;
+        writer = nextWriter();
+    }
+
+    /**
+     * This is called when we want to rotate the output file to
+     * start writing the next file.  There are two scenarios when
+     * we do this:
+     *
+     * <UL>
+     *  <LI> initial case, when we have no file
+     *  <LI> when we have filled the file and want to rotate it
+     * </UL>
+     *
+     */
+    private Writer nextWriter () throws IOException {
+
+        if (writer != null) {
+            writer.close();
+        }
+
+        int maxAttempts = 1000;
+        while (maxAttempts-- > 0) {
+            String name = prefix + "-" + generation++;
+            File f = new File(name);
+
+            // make sure directory exists
+            File dir = f.getParentFile();
+            if (! dir.exists()) {
+                dir.mkdirs();
+            }
+
+            // if compressed version exists we skip it
+            if ((new File(name + ".gz").exists())
+                || (new File(name + ".bz2").exists())) {
+                continue;
+            }
+
+            // if file does not exist we have a winner
+            if (! f.exists()) {
+                log.log(LogLevel.DEBUG, "nextWriter, new file: " + name);
+                currentFile = f;
+                bytesWritten = 0;
+                return new FileWriter(f, true);
+            }
+
+            // just skip over directories for now
+            if (! f.isFile()) {
+                log.fine("nextWriter, " + name + " is a directory, skipping");
+                continue;
+            }
+
+            // if the size is < resumeSizeLimit then we open it
+            if (f.length() < resumeLimitSize) {
+                log.fine("nextWriter, resuming " + name + ", length was " + f.length());
+                currentFile = f;
+                bytesWritten = f.length();
+                return new FileWriter(f, true);
+            } else {
+
+                log.fine("nextWriter, not resuming " + name
+                         + " because it is bigger than "
+                         + resumeLimit
+                         + " percent of max");
+            }
+        }
+
+        throw new RuntimeException("Unable to create next log file");
+    }
+
+    /**
+     * Note that this method should not be used directly since
+     * that would circumvent rotation when it grows past its
+     * maximum size.  use the one that takes String instead.
+     *
+     * <P>
+     * <em>
+     * (This is a class which is only used internally anyway)
+     * </em>
+     */
+    public void write (char[] cbuff, int offset, int len) throws IOException {
+        throw new RuntimeException("This method should not be used");
+    }
+
+    public void write(String str) throws IOException {
+        if (writer == null) {
+            writer = nextWriter();
+        }
+
+        bytesWritten += str.length();
+        writer.write(str, 0, str.length());
+
+        if (bytesWritten >= maxSize) {
+            log.fine("logfile '"
+                     + currentFile.getAbsolutePath()
+                     + "' full, rotating");
+            writer = nextWriter();
+        }
+    }
+
+
+    public void flush() throws IOException{
+        if (writer != null) {
+            writer.flush();
+        }
+    }
+
+    public void close() throws IOException {
+        flush();
+        if (writer != null) {
+            writer.close();
+            writer = null;
+        }
+    }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java
new file mode 100644
index 00000000000..21ca26efdd2
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.archive;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+import java.util.logging.Level;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author  <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+@SuppressWarnings("serial")
+public class LogWriterLRUCache extends LinkedHashMap<Integer, LogWriter>
+{
+    private static final Logger log
+        = Logger.getLogger(LogWriterLRUCache.class.getName());
+
+    final int maxEntries = 100;
+
+    public LogWriterLRUCache(int initialCapacity,
+                             float loadFactor) {
+        super(initialCapacity, loadFactor, true);
+    }
+
+    // TODO: implement unit test for this
+    protected boolean removeEldestEntry (Map.Entry<Integer, LogWriter> eldest) {
+        if (size() > maxEntries) {
+            LogWriter logWriter = (LogWriter)eldest.getValue();
+            log.fine("Closing oldest LogWriter: " + logWriter);
+            try {
+                logWriter.close();
+            } catch (IOException e) {
+                log.log(Level.WARNING, "closing LogWriter failed", e);
+            }
+            return true;
+        }
+
+        return false;
+    }
+}