From 8e0adc45fe6fb1f90e77295ab0b89095ed300530 Mon Sep 17 00:00:00 2001 From: Eirik Nygaard Date: Thu, 16 Jun 2016 12:28:37 +0200 Subject: Add missing logserver files --- .../handlers/archive/ArchiverHandler.java | 255 +++++++++++++++++++++ .../logserver/handlers/archive/ArchiverPlugin.java | 77 +++++++ .../logserver/handlers/archive/LogWriter.java | 147 ++++++++++++ .../handlers/archive/LogWriterLRUCache.java | 41 ++++ 4 files changed, 520 insertions(+) create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java (limited to 'logserver/src') diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java new file mode 100644 index 00000000000..06bf5183005 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -0,0 +1,255 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.archive; + +import java.io.File; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.Iterator; +import java.util.TimeZone; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.yahoo.logserver.filter.LogFilter; +import com.yahoo.logserver.filter.LogFilterManager; + +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.handlers.AbstractLogHandler; + + +/** + * This class implements a log handler which archives the incoming + * messages based on their timestamp. The goal of this archiver + * is to make it easy to locate messages in a time interval, while + * ensuring that no log file exceeds the maximum allowed size. + * + *

+ * This class is not thread safe. + *

+ * + *

+ * TODO: + *

+ * + * @author Bjorn Borud + */ +public class ArchiverHandler extends AbstractLogHandler +{ + private static final Logger log + = Logger.getLogger(ArchiverHandler.class.getName()); + + /** File instance representing root directory for logging */ + private File root; + + /** Root directory for logging */ + private String absoluteRootDir; + + /** Max number of log files open at any given time */ + private final int maxFilesOpen = 100; + + /** + *The maximum number of bytes we allow a file to grow to + * before we rotate it + */ + private int maxFileSize; + + /** Calendar instance for operating on Date objects */ + private final Calendar calendar; + + /** DateFormat instance for building filenames */ + private final SimpleDateFormat dateformat; + + /** + * This is an LRU cache for LogWriter objects. Remember that + * we have one LogWriter for each time slot + */ + private final LogWriterLRUCache logWriterLRUCache; + + /** + * Filtering + */ + private LogFilter filter = null; + + /** + * Creates an ArchiverHandler which puts files under + * the given root directory. + */ + public ArchiverHandler () { + calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + dateformat = new SimpleDateFormat("yyyy/MM/dd/HH"); + dateformat.setTimeZone(TimeZone.getTimeZone("UTC")); + + // Set up filtering + String archiveMetrics = System.getProperty("vespa_log_server__archive_metric"); + if ("off".equals(archiveMetrics)){ + filter = LogFilterManager.getLogFilter("system.nometricsevents"); + } + + setLogFilter(filter); + + // set up LRU for files + logWriterLRUCache = new LogWriterLRUCache(maxFilesOpen, + (float)0.75); + } + + /** + * Creates an ArchiverHandler which puts files under + * the given root directory. + */ + public ArchiverHandler (String rootDir, int maxFileSize) { + this(); + setRootDir(rootDir); + this.maxFileSize = maxFileSize; + } + + + /** + * Return the appropriate LogWriter given a log message. + */ + private synchronized LogWriter getLogWriter (LogMessage m) throws IOException { + Integer slot = new Integer(dateHash(m.getTime())); + LogWriter logWriter = logWriterLRUCache.get(slot); + if (logWriter != null) { + return logWriter; + } + + // invariant: LogWriter we sought was not in the cache + logWriter = new LogWriter(getPrefix(m), maxFileSize); + logWriterLRUCache.put(slot, logWriter); + + return logWriter; + } + + /** + * This method is just a fancy way of generating a stripped + * down number representing year, month, day and hour in order + * to partition logging in time. + * + *

+ * This method is not thread-safe. + */ + public int dateHash (long time) { + calendar.setTimeInMillis(time); + int year = calendar.get(Calendar.YEAR); + int month = calendar.get(Calendar.MONTH) + 1; + int day = calendar.get(Calendar.DAY_OF_MONTH); + int hour = calendar.get(Calendar.HOUR_OF_DAY); + + return year * 1000000 + + month * 10000 + + day * 100 + + hour; + } + + /** + * Generate prefix for log filenames based on log message. + * + *

+ * This message is public only because we need + * access to it in unit tests. For all practical purposes this + * method does not exist to you, the application programmer, OK? :-) + *

+ * XXX optimize! + */ + public String getPrefix (LogMessage msg) { + calendar.setTimeInMillis(msg.getTime()); +/* + int year = calendar.get(Calendar.YEAR); + int month = calendar.get(Calendar.MONTH) + 1; + int day = calendar.get(Calendar.DAY_OF_MONTH); + int hour = calendar.get(Calendar.HOUR_OF_DAY); +*/ + StringBuffer result = new StringBuffer(absoluteRootDir.length() + + 1 // slash + + 4 // year + + 1 // slash + + 2 // month + + 1 // slash + + 2 // day + + 1 // slash + + 2 // hour + ) + .append(absoluteRootDir).append("/") + .append(dateformat.format(calendar.getTime())); + return result.toString(); + } + + public boolean doHandle (LogMessage msg) { + try { + LogWriter logWriter = getLogWriter(msg); + logWriter.write(msg.toString()); + } + catch (IOException e) { + throw new RuntimeException(e); + } + return true; + } + + public synchronized void flush () { + for (LogWriter l : logWriterLRUCache.values()) { + try { + l.flush(); + } + catch (IOException e) { + log.log(Level.WARNING, "Flushing failed", e); + } + } + } + + public synchronized void close () { + Iterator it = logWriterLRUCache.values().iterator(); + while (it.hasNext()) { + LogWriter l = it.next(); + try { + l.close(); + } + catch (IOException e) { + log.log(Level.WARNING, "Closing failed", e); + } + it.remove(); + } + } + + private void setRootDir(String rootDir) { + // roundabout way of setting things, but this way we can + // get around Java's ineptitude for file handling. (relative + // paths in File are broken) + // + absoluteRootDir = new File(rootDir).getAbsolutePath(); + root = new File(absoluteRootDir); + + // ensure that root dir exists + if (root.isDirectory()) { + log.log(LogLevel.DEBUG, "Using " + absoluteRootDir + " as root"); + } else { + if (! root.mkdirs()) { + log.log(LogLevel.ERROR, + "Unable to create directory " + absoluteRootDir); + } else { + log.log(LogLevel.DEBUG, "Created root at " + absoluteRootDir); + } + } + + } + + public String toString () { + return ArchiverHandler.class.getName() + ": root=" + absoluteRootDir; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java new file mode 100644 index 00000000000..0d04b4cf981 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.archive; + +import java.util.logging.Logger; +import com.yahoo.logserver.Server; +import com.yahoo.plugin.Config; +import com.yahoo.plugin.Plugin; + + +public class ArchiverPlugin implements Plugin +{ + /** + * Default log archive dir (relative to current directory + * at startup). + */ + private static final String DEFAULT_DIR = "logarchive"; + + /** + * Default max file size for archived log files. + */ + private static final String DEFAULT_MAXFILESIZE = "20971520"; + + private final Server server = Server.getInstance(); + private static final Logger log = + Logger.getLogger(ArchiverPlugin.class.getName()); + private ArchiverHandler archiver; + + /** + * @return the name of this plugin + */ + public String getPluginName() { + return "logarchive"; + } + + /** Initialize the archiver plugin + * + * Config keys used: + * + * maxfilesize + * dir The root of the logarchive, make sure this does + * not end with a '/' character. + */ + public void initPlugin(Config config) { + + if (archiver != null) { + log.finer("ArchivePlugin doubly initialized"); + throw new IllegalStateException("plugin already initialized: " + + getPluginName()); + } + + // Possible to disable logarchive for testing + String rootDir = config.get("dir", DEFAULT_DIR); + int maxFileSize = config.getInt("maxfilesize", DEFAULT_MAXFILESIZE); + String threadName = config.get("thread", getPluginName()); + + // register log handler and flusher + archiver = new ArchiverHandler(rootDir, maxFileSize); + server.registerLogHandler(archiver, threadName); + server.registerFlusher(archiver); + } + + /** + * Shut down the archiver plugin. + */ + public void shutdownPlugin() { + + if (archiver == null) { + log.finer("ArchiverPlugin shutdown before initialize"); + throw new IllegalStateException("plugin not initialized: " + + getPluginName()); + } + server.unregisterLogHandler(archiver); + server.unregisterFlusher(archiver); + archiver.close(); + archiver = null; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java new file mode 100644 index 00000000000..fb78e8243c3 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java @@ -0,0 +1,147 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.archive; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.util.logging.Logger; + +import com.yahoo.log.LogLevel; + +/** + * This class is not thread-safe. + * + * + * @author Bjorn Borud + */ +public class LogWriter extends Writer +{ + private static final Logger log = Logger.getLogger(LogWriter.class.getName()); + + private long bytesWritten = 0; + private int generation = 0; + private int maxSize = 20 * (1024*1024); + private final int resumeLimit = 95; + private final int resumeLimitSize = (maxSize * resumeLimit / 100); + private File currentFile; + private Writer writer; + private final String prefix; + + public LogWriter (String prefix, int maxSize) throws IOException { + this.prefix = prefix; + this.maxSize = maxSize; + writer = nextWriter(); + } + + /** + * This is called when we want to rotate the output file to + * start writing the next file. There are two scenarios when + * we do this: + * + *

+ * + */ + private Writer nextWriter () throws IOException { + + if (writer != null) { + writer.close(); + } + + int maxAttempts = 1000; + while (maxAttempts-- > 0) { + String name = prefix + "-" + generation++; + File f = new File(name); + + // make sure directory exists + File dir = f.getParentFile(); + if (! dir.exists()) { + dir.mkdirs(); + } + + // if compressed version exists we skip it + if ((new File(name + ".gz").exists()) + || (new File(name + ".bz2").exists())) { + continue; + } + + // if file does not exist we have a winner + if (! f.exists()) { + log.log(LogLevel.DEBUG, "nextWriter, new file: " + name); + currentFile = f; + bytesWritten = 0; + return new FileWriter(f, true); + } + + // just skip over directories for now + if (! f.isFile()) { + log.fine("nextWriter, " + name + " is a directory, skipping"); + continue; + } + + // if the size is < resumeSizeLimit then we open it + if (f.length() < resumeLimitSize) { + log.fine("nextWriter, resuming " + name + ", length was " + f.length()); + currentFile = f; + bytesWritten = f.length(); + return new FileWriter(f, true); + } else { + + log.fine("nextWriter, not resuming " + name + + " because it is bigger than " + + resumeLimit + + " percent of max"); + } + } + + throw new RuntimeException("Unable to create next log file"); + } + + /** + * Note that this method should not be used directly since + * that would circumvent rotation when it grows past its + * maximum size. use the one that takes String instead. + * + *

+ * + * (This is a class which is only used internally anyway) + * + */ + public void write (char[] cbuff, int offset, int len) throws IOException { + throw new RuntimeException("This method should not be used"); + } + + public void write(String str) throws IOException { + if (writer == null) { + writer = nextWriter(); + } + + bytesWritten += str.length(); + writer.write(str, 0, str.length()); + + if (bytesWritten >= maxSize) { + log.fine("logfile '" + + currentFile.getAbsolutePath() + + "' full, rotating"); + writer = nextWriter(); + } + } + + + public void flush() throws IOException{ + if (writer != null) { + writer.flush(); + } + } + + public void close() throws IOException { + flush(); + if (writer != null) { + writer.close(); + writer = null; + } + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java new file mode 100644 index 00000000000..21ca26efdd2 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.archive; + +import java.io.IOException; +import java.util.logging.Logger; +import java.util.logging.Level; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * @author Bjorn Borud + */ +@SuppressWarnings("serial") +public class LogWriterLRUCache extends LinkedHashMap +{ + private static final Logger log + = Logger.getLogger(LogWriterLRUCache.class.getName()); + + final int maxEntries = 100; + + public LogWriterLRUCache(int initialCapacity, + float loadFactor) { + super(initialCapacity, loadFactor, true); + } + + // TODO: implement unit test for this + protected boolean removeEldestEntry (Map.Entry eldest) { + if (size() > maxEntries) { + LogWriter logWriter = (LogWriter)eldest.getValue(); + log.fine("Closing oldest LogWriter: " + logWriter); + try { + logWriter.close(); + } catch (IOException e) { + log.log(Level.WARNING, "closing LogWriter failed", e); + } + return true; + } + + return false; + } +} -- cgit v1.2.3