summaryrefslogtreecommitdiffstats
path: root/logserver
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /logserver
Publish
Diffstat (limited to 'logserver')
-rw-r--r--logserver/.gitignore16
-rw-r--r--logserver/OWNERS1
-rwxr-xr-xlogserver/bin/logserver-start.sh75
-rw-r--r--logserver/pom.xml87
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java46
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java31
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/Flusher.java57
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java188
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/PluginLoader.java22
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/Server.java208
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/demo/.gitignore0
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java30
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java23
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java99
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java50
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java30
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java21
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java19
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java28
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java70
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java29
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java51
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java137
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java241
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java51
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java186
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java141
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java51
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java233
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java54
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java78
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java130
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java411
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java58
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java272
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java41
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java125
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/net/control/State.java54
-rw-r--r--logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java65
-rw-r--r--logserver/src/main/java/com/yahoo/plugin/Config.java23
-rw-r--r--logserver/src/main/java/com/yahoo/plugin/Plugin.java33
-rw-r--r--logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java35
-rw-r--r--logserver/src/test/files/event.txt.gzbin0 -> 11272 bytes
-rw-r--r--logserver/src/test/files/logEntries.txt3
-rw-r--r--logserver/src/test/files/value-events.txt982
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java25
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java55
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java43
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java45
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java56
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java36
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java30
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java46
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java96
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java115
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java122
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java43
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java43
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java134
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/handlers/status/.gitignore0
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/httphandlers/.gitignore0
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java92
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java211
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java131
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java64
-rw-r--r--logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java53
66 files changed, 6025 insertions, 0 deletions
diff --git a/logserver/.gitignore b/logserver/.gitignore
new file mode 100644
index 00000000000..3c9aa00964f
--- /dev/null
+++ b/logserver/.gitignore
@@ -0,0 +1,16 @@
+.classpath
+.git
+.project
+archive
+build
+classes
+javamake.pdb
+log
+logarchive
+logserver.iws
+plugins
+staging
+target
+testLogs
+vespatest_dump
+/pom.xml.build
diff --git a/logserver/OWNERS b/logserver/OWNERS
new file mode 100644
index 00000000000..67cd2820bb8
--- /dev/null
+++ b/logserver/OWNERS
@@ -0,0 +1 @@
+arnej27959
diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh
new file mode 100755
index 00000000000..96dac54aafe
--- /dev/null
+++ b/logserver/bin/logserver-start.sh
@@ -0,0 +1,75 @@
+#!/bin/sh
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+ myname=${0}
+ mypath=${myname%/*}
+ myname=${myname##*/}
+ if [ "$mypath" ] && [ -d "$mypath" ]; then
+ return
+ fi
+ mypath=$(pwd)
+ if [ -f "${mypath}/${myname}" ]; then
+ return
+ fi
+ echo "FATAL: Could not figure out the path where $myname lives from $0"
+ exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+ if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+ # ensure it ends with "/" :
+ VESPA_HOME=${VESPA_HOME%/}/
+ export VESPA_HOME
+ common_env=$VESPA_HOME/$COMMON_ENV
+ if [ -f "$common_env" ]; then
+ . $common_env
+ return
+ fi
+ fi
+ return 1
+}
+
+findroot () {
+ source_common_env && return
+ if [ "$VESPA_HOME" ]; then
+ echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+ exit 1
+ fi
+ if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+ VESPA_HOME="$ROOT"
+ source_common_env && return
+ fi
+ findpath
+ while [ "$mypath" ]; do
+ VESPA_HOME=${mypath}
+ source_common_env && return
+ mypath=${mypath%/*}
+ done
+ echo "FATAL: missing VESPA_HOME environment variable"
+ echo "Could not locate $COMMON_ENV anywhere"
+ exit 1
+}
+
+findroot
+
+# END environment bootstrap section
+
+ROOT=$VESPA_HOME
+export ROOT
+cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; }
+
+addopts="-server -Xms64m -Xmx256m -XX:MaxDirectMemorySize=76m -XX:MaxJavaStackTraceDepth=-1"
+
+oomopt="-XX:OnOutOfMemoryError=kill -9 %p"
+
+jar="-jar $ROOT/lib/jars/logserver-jar-with-dependencies.jar"
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+
+exec java $addopts "$oomopt" "$@" $jar
diff --git a/logserver/pom.xml b/logserver/pom.xml
new file mode 100644
index 00000000000..dc997db0386
--- /dev/null
+++ b/logserver/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>6-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+ <artifactId>logserver</artifactId>
+ <packaging>jar</packaging>
+ <version>6-SNAPSHOT</version>
+ <name>logserver</name>
+ <description>Log server.</description>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespajlib</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespalog</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <compilerArgs>
+ <arg>-Xlint:rawtypes</arg>
+ <arg>-Xlint:unchecked</arg>
+ <arg>-Xlint:deprecation</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>com.yahoo.logserver.Server</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <!-- append to the packaging phase. -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-install-plugin</artifactId>
+ <configuration>
+ <updateReleaseInfo>true</updateReleaseInfo>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java
new file mode 100644
index 00000000000..6d215b9e831
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.plugin.Plugin;
+import com.yahoo.plugin.SystemPropertyConfig;
+
+import java.util.logging.Logger;
+
+/**
+ * TODO: describe class
+ *
+ * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a>
+ */
+public abstract class AbstractPluginLoader implements PluginLoader {
+ private static final Logger log = Logger.getLogger(AbstractPluginLoader.class.getName());
+
+ public abstract void loadPlugins();
+
+ protected void loadFromClass (Class<? extends Plugin> pluginClass) {
+ Plugin plugin;
+ try {
+ plugin = (Plugin)pluginClass.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ log.log(LogLevel.ERROR, pluginClass.getName() + ": load failed: " + e);
+ throw new RuntimeException(e);
+ }
+
+ String pname = plugin.getPluginName();
+ String prefix = Server.APPNAME + "." + pname + ".";
+ SystemPropertyConfig config = new SystemPropertyConfig(prefix);
+ String enable = config.get("enable", "true");
+
+ if (!enable.equals("true")) {
+ log.log(LogLevel.INFO, pname + ": plugin disabled by config");
+ return;
+ }
+
+ try {
+ plugin.initPlugin(config);
+ log.log(LogLevel.DEBUG, pname + ": plugin loaded");
+ } catch (Exception e) {
+ log.log(LogLevel.ERROR, pname + ": init failed", e);
+ }
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java
new file mode 100644
index 00000000000..f2ef60267fe
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver;
+
+import java.util.logging.Logger;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.logserver.handlers.archive.ArchiverPlugin;
+import com.yahoo.logserver.handlers.lasterrorsholder.LastErrorsHolderPlugin;
+import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin;
+import com.yahoo.logserver.handlers.replicator.ReplicatorPlugin;
+
+/**
+ * Load a set of builtin plugins
+ *
+ * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a>
+ */
+public class BuiltinPluginLoader extends AbstractPluginLoader {
+ private static final Logger log = Logger.getLogger(BuiltinPluginLoader.class.getName());
+
+ public void loadPlugins() {
+ log.log(LogLevel.DEBUG, "starting to load builtin plugins");
+
+ loadFromClass(ArchiverPlugin.class);
+ loadFromClass(ReplicatorPlugin.class);
+ loadFromClass(LogMetricsPlugin.class);
+ loadFromClass(LastErrorsHolderPlugin.class);
+
+ log.log(LogLevel.DEBUG, "done loading builtin plugins");
+ }
+
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/Flusher.java b/logserver/src/main/java/com/yahoo/logserver/Flusher.java
new file mode 100644
index 00000000000..ee2b317f573
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/Flusher.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver;
+
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.logging.Logger;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.logserver.handlers.LogHandler;
+
+/**
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class Flusher extends Thread {
+ private static final Logger log = Logger.getLogger(Flusher.class.getName());
+ private static final Flusher instance;
+ private static final List<WeakReference<LogHandler>> logHandlers =
+ new ArrayList<WeakReference<LogHandler>>();
+
+ static {
+ instance = new Flusher();
+ instance.start();
+ }
+
+ Flusher() {
+ super("flusher");
+ }
+
+ public static synchronized void register(LogHandler logHandler) {
+ logHandlers.add(new WeakReference<LogHandler>(logHandler));
+ }
+
+ public synchronized void run() {
+ try {
+ while(!isInterrupted()) {
+ Thread.sleep(2000);
+ Iterator<WeakReference<LogHandler>> it = logHandlers.iterator();
+ while (it.hasNext()) {
+ WeakReference<LogHandler> r = it.next();
+ LogHandler h = r.get();
+ if (h == null) {
+ it.remove();
+ } else {
+ h.flush();
+ }
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Flushing " + h);
+ }
+ }
+ }
+ } catch (InterruptedException e) {
+ log.log(LogLevel.WARNING, "flusher was interrupted", e);
+ }
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java
new file mode 100644
index 00000000000..6de34d4a899
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java
@@ -0,0 +1,188 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+import com.yahoo.io.SelectLoopHook;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.handlers.LogHandler;
+
+
+/**
+ * This is the central point from which LogMessage objects are
+ * propagated throughout the logserver architecture.
+ *
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class LogDispatcher implements LogHandler, SelectLoopHook
+{
+ private static final Logger log = Logger.getLogger(LogDispatcher.class.getName());
+
+ private final List<LogHandler> handlers = new ArrayList<LogHandler>();
+ private int messageCount = 0;
+ private boolean hasBeenShutDown = false;
+ private boolean batchedMode = false;
+ private final int batchSize = 5000;
+ private List<LogMessage> currentBatchList;
+ private int roundCount = 0;
+ @SuppressWarnings("unused")
+ private int lastRoundCount = 0;
+
+ public LogDispatcher () {
+ }
+
+ /**
+ * Dispatches a message to all the LogHandler instances we've
+ * got registered. The main entry point for LogMessage instances
+ * into the log server.
+ *
+ * @param msg The LogMessage instance we wish to dispatch to the
+ * plugins
+ */
+ public synchronized void handle (LogMessage msg) {
+ if (msg == null) {
+ throw new NullPointerException("LogMessage was null");
+ }
+
+ if (batchedMode) {
+ addToBatch(msg);
+ } else {
+ for (LogHandler h : handlers) {
+ h.handle(msg);
+ }
+ }
+ messageCount++;
+ }
+
+ private void addToBatch (LogMessage msg) {
+ if (currentBatchList == null) {
+ currentBatchList = new ArrayList<LogMessage>(batchSize);
+ currentBatchList.add(msg);
+ return;
+ }
+
+ currentBatchList.add(msg);
+
+ if (currentBatchList.size() == batchSize) {
+ flushBatch();
+ }
+ }
+
+ private void flushBatch () {
+ if (currentBatchList == null) {
+ return;
+ }
+
+ for (LogHandler ht : handlers) {
+ ht.handle(currentBatchList);
+ }
+ currentBatchList = null;
+ }
+
+
+ public void handle (List<LogMessage> messages) {
+ throw new IllegalStateException("method not supported");
+ }
+
+ /**
+ * Set the batched mode. Note that this should only be set
+ * at initialization time because it radically changes the
+ * behavior of the dispatcher. When in batched mode, the
+ * dispatcher will not enqueue single LogMessage instances
+ * but lists of same.
+ */
+ public void setBatchedMode (boolean batchedMode) {
+ this.batchedMode = batchedMode;
+ }
+
+ public synchronized void flush () {
+ if (batchedMode) {
+ flushBatch();
+ }
+
+ for (LogHandler h : handlers) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Flushing " + h.toString());
+ }
+ h.flush();
+ }
+ }
+
+ public synchronized void close () {
+ if (hasBeenShutDown) {
+ throw new IllegalStateException("Shutdown already in progress");
+ }
+ hasBeenShutDown = true;
+
+ for (LogHandler ht : handlers) {
+ if (ht instanceof Thread) {
+ log.fine("Stopping " + ht);
+ ((Thread)ht).interrupt();
+ }
+ }
+ handlers.clear();
+
+ log.log(LogLevel.DEBUG, "Logdispatcher shut down. Handled " + messageCount + " messages");
+ }
+
+ /**
+ * Register handler thread with the dispatcher. If the handler
+ * thread has already been registered, we log a warning and
+ * just do nothing.
+ *
+ * <P>
+ * If the thread is not alive it will be start()'ed.
+ *
+ */
+ public synchronized void registerLogHandler (LogHandler ht) {
+ if (hasBeenShutDown) {
+ throw new IllegalStateException("Tried to register LogHandler on"+
+ " LogDispatcher which was shut down");
+ }
+
+ if (handlers.contains(ht)) {
+ log.warning("LogHandler was already registered: " + ht);
+ return;
+ }
+ handlers.add(ht);
+
+ if ((ht instanceof Thread) && (! ((Thread)ht).isAlive())) {
+ ((Thread)ht).start();
+ }
+
+ log.fine("Added (and possibly started) LogHandler " + ht);
+ }
+
+ /**
+ * Make defensive copy and return array of LogHandlers.
+ */
+ public LogHandler[] getLogHandlers () {
+ LogHandler[] h = new LogHandler[handlers.size()];
+ return handlers.toArray(h);
+ }
+
+ /**
+ * Return message counter.
+ *
+ * @return Returns the number of messages that we have seen.
+ */
+ public synchronized int getMessageCount () {
+ return messageCount;
+ }
+
+ /**
+ * Hook which is called when the select loop has finished.
+ */
+ public void selectLoopHook (boolean before) {
+ if (batchedMode) {
+ flushBatch();
+ }
+
+ lastRoundCount = messageCount - roundCount;
+ roundCount = messageCount;
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/PluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/PluginLoader.java
new file mode 100644
index 00000000000..2357eefb804
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/PluginLoader.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver;
+
+/**
+ * This interface specifies an API for implementing logserver plugin
+ * loaders. A plugin loader has two basic tasks: to load or unload
+ * all of its knows plugins. In addition, if a plugin loader's
+ * canReload() method returns <code>true</code>, plugins may be loaded
+ * again after they are unloaded.
+ *
+ * <p> Plugins loaded through such reload-capable plugin loaders may
+ * be upgraded without restarting the server.
+ *
+ * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a>
+ */
+public interface PluginLoader
+{
+ /**
+ * Load all plugins known to this loader.
+ */
+ public void loadPlugins();
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/Server.java b/logserver/src/main/java/com/yahoo/logserver/Server.java
new file mode 100644
index 00000000000..d09ae305dc2
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/Server.java
@@ -0,0 +1,208 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver;
+
+import com.yahoo.io.FatalErrorHandler;
+import com.yahoo.io.Listener;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogSetup;
+import com.yahoo.log.event.Event;
+import com.yahoo.logserver.handlers.HandlerThread;
+import com.yahoo.logserver.handlers.LogHandler;
+import com.yahoo.logserver.net.LogConnectionFactory;
+import com.yahoo.logserver.net.control.Levels;
+import com.yahoo.system.CatchSigTerm;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * This class implements the log server itself. At present there is
+ * no runtime configuration; the server starts up, binds port 19081
+ * and loads builtin plugins.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a>
+ */
+
+public class Server implements Runnable {
+ private final AtomicBoolean signalCaught = new AtomicBoolean(false);
+ public static final String APPNAME = "logserver";
+ private static final Server instance = new Server();
+ private static final Logger log = Logger.getLogger(Server.class.getName());
+ private static final FatalErrorHandler fatalErrorHandler =
+ new FatalErrorHandler();
+ private static final HashMap<String,HandlerThread> handlerThreads =
+ new HashMap<String,HandlerThread>();
+ private static final HashMap<LogHandler,String> threadNameForHandler =
+ new HashMap<LogHandler,String>();
+
+ static {
+ LogSetup.initVespaLogging("ADM");
+ }
+
+ // the port is a String because we want to use it as the default
+ // value of a System.getProperty().
+ private static final String LISTEN_PORT = "19081";
+
+ private int listenPort;
+ private Listener listener;
+ private final LogDispatcher dispatch;
+
+ private final boolean isInitialized;
+
+ /**
+ * Server constructor
+ */
+ private Server () {
+ dispatch = new LogDispatcher();
+ dispatch.setBatchedMode(true);
+ isInitialized = false;
+ }
+
+ public static Server getInstance () {
+ return instance;
+ }
+
+ private HandlerThread getHandlerThread (String threadName) {
+ threadName += " handler thread";
+ HandlerThread ht = handlerThreads.get(threadName);
+ if (ht == null) {
+ ht = new HandlerThread(threadName);
+ handlerThreads.put(threadName, ht);
+ ht.setFatalErrorHandler(fatalErrorHandler);
+ dispatch.registerLogHandler(ht);
+ }
+ return ht;
+ }
+
+ public void registerPluginLoader (PluginLoader loader) {
+ loader.loadPlugins();
+ }
+
+ public void registerLogHandler (LogHandler lh, String threadName) {
+ HandlerThread ht = getHandlerThread(threadName);
+ ht.registerHandler(lh);
+ threadNameForHandler.put(lh, threadName);
+ }
+
+ public void unregisterLogHandler (LogHandler lh) {
+ String threadName = threadNameForHandler.get(lh);
+ unregisterLogHandler(lh, threadName);
+ }
+
+ public void unregisterLogHandler (LogHandler lh, String threadName) {
+ HandlerThread ht = getHandlerThread(threadName);
+ ht.unregisterHandler(lh);
+ threadNameForHandler.remove(lh);
+ }
+
+ public void registerFlusher (LogHandler lh) {
+ Flusher.register(lh);
+ }
+
+ /** Included only for consistency */
+ public void unregisterFlusher (LogHandler lh) {
+ /* NOP */
+ }
+
+ /**
+ * Initialize the server and start up all its plugins,
+ *
+ * @param listenPort The port on which the logserver accepts log
+ * messages.
+ */
+ public void initialize (int listenPort) {
+ if (isInitialized) {
+ throw new IllegalStateException(APPNAME + " already initialized");
+ }
+
+ this.listenPort = listenPort;
+
+ // plugins
+ registerPluginLoader(new BuiltinPluginLoader());
+
+ // main listener
+ listener = new Listener(APPNAME);
+ listener.addSelectLoopPostHook(dispatch);
+ listener.setFatalErrorHandler(fatalErrorHandler);
+ }
+
+ /**
+ * Sets up the listen port and starts the Listener. Then waits for
+ * Listener to exit.
+ */
+ public void run() {
+ try {
+ listener.listen(new LogConnectionFactory(dispatch), listenPort);
+ log.log(LogLevel.CONFIG, APPNAME + ".listenport=" + listenPort);
+ } catch (IOException e) {
+ log.log(LogLevel.ERROR, "Unable to initialize", e);
+ return;
+ }
+
+ log.fine("Starting listener...");
+ listener.start();
+ Event.started(APPNAME);
+ try {
+ listener.join();
+ log.fine("listener thread exited");
+ }
+ catch (InterruptedException e) {
+ log.log(Level.WARNING, "Server was interrupted", e);
+ }
+ }
+ private void setupSigTermHandler() {
+ CatchSigTerm.setup(signalCaught); // catch termination signal
+ }
+
+ private void waitForShutdown() {
+ synchronized (signalCaught) {
+ while (!signalCaught.get()) {
+ try {
+ signalCaught.wait();
+ } catch (InterruptedException e) {
+ // empty
+ }
+ }
+ }
+ Event.stopping(APPNAME, "shutdown");
+ dispatch.close();
+ Event.stopped(APPNAME, 0, 0);
+ System.exit(0);
+ }
+
+ public static HashMap<LogHandler, String> threadNameForHandler() {
+ return threadNameForHandler;
+ }
+
+ public static void help () {
+ System.out.println();
+ System.out.println("System properties:");
+ System.out.println(" - " + APPNAME + ".listenport (" + LISTEN_PORT + ")");
+ System.out.println(" - " + APPNAME + ".queue.size ("
+ + HandlerThread.DEFAULT_QUEUESIZE
+ + ")");
+ System.out.println(" - logserver.default.loglevels ("
+ + (new Levels()).toString()
+ + ")");
+ System.out.println();
+ }
+
+ public static void main (String[] args) {
+ if (args.length > 0 && "-help".equals(args[0])) {
+ help();
+ System.exit(0);
+ }
+
+ String portString = System.getProperty(APPNAME + ".listenport", LISTEN_PORT);
+ Server server = Server.getInstance();
+ server.setupSigTermHandler();
+ server.initialize(Integer.parseInt(portString));
+
+ Thread t = new Thread(server, "logserver main");
+ t.start();
+ server.waitForShutdown();
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/demo/.gitignore b/logserver/src/main/java/com/yahoo/logserver/demo/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/demo/.gitignore
diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java
new file mode 100644
index 00000000000..b229af522ab
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.filter;
+
+import com.yahoo.log.LogMessage;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.logging.Level;
+/**
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class LevelFilter implements LogFilter {
+ private final Set<Level> levels = new HashSet<Level>();
+
+ public void addLevel (Level level) {
+ levels.add(level);
+ }
+
+ public void removeLevel (Level level) {
+ levels.remove(level);
+ }
+
+ public boolean isLoggable (LogMessage msg) {
+ return levels.contains(msg.getLevel());
+ }
+
+ public String description () {
+ return "Match specific log levels";
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java
new file mode 100644
index 00000000000..8c4c3911c9e
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.filter;
+
+import com.yahoo.log.LogMessage;
+
+/**
+ * This interface is analogous to the java.util.logging.Filter
+ * interface. Classes implementing this interface should be
+ * <b>stateless/immutable if possible so filters can be
+ * shared</b>.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public interface LogFilter {
+ /**
+ * Determine if this log message is loggable.
+ *
+ * @param msg The log message
+ *
+ */
+ public boolean isLoggable (LogMessage msg);
+ public String description ();
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java
new file mode 100644
index 00000000000..a68aff86e10
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.filter;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.yahoo.log.LogLevel;
+
+/**
+ * The LogFilterManager keeps track of associations between
+ * LogFilter names and instances, so that access to filters
+ * is truly global. It also manages the LogFilter namespace
+ * to ensure that system-defined filters are not tampered with.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+
+public class LogFilterManager {
+ private static final LogFilterManager instance;
+
+ static {
+ instance = new LogFilterManager();
+
+ LevelFilter allEvents = new LevelFilter();
+ allEvents.addLevel(LogLevel.EVENT);
+ instance.addLogFilterInternal("system.allevents", allEvents);
+ instance.addLogFilterInternal("system.metricsevents", new MetricsFilter());
+ instance.addLogFilterInternal("system.nometricsevents", new NoMetricsFilter());
+ instance.addLogFilterInternal("system.all", new NullFilter());
+ instance.addLogFilterInternal("system.mute", MuteFilter.getInstance());
+ }
+
+ private final Map<String,LogFilter> filters = new HashMap<String,LogFilter>();
+
+ private LogFilterManager () {
+ }
+
+ /**
+ * Public interface for adding a name-logfilter mapping. If
+ * there exists a mapping already the old mapping is replaced
+ * with the new mapping.
+ *
+ * If the name is within the namespace reserved for internal
+ * built-in filters it will throw an exception
+ */
+ public static void addLogFilter (String name, LogFilter filter) {
+ if (filter == null) {
+ throw new NullPointerException("filter cannot be null");
+ }
+
+ if (name == null) {
+ throw new NullPointerException("name cannot be null");
+ }
+
+ String n = name.toLowerCase();
+
+ if (n.startsWith("system.")) {
+ throw new IllegalArgumentException("'system' namespace is reserved");
+ }
+
+ instance.addLogFilterInternal(n, filter);
+ }
+
+ /**
+ * LogFilter lookup function
+ *
+ * @param name The name of the LogFilter to be looked up.
+ * @return Returns the LogFilter associated with this name or
+ * <code>null</code> if not found.
+ */
+ public static LogFilter getLogFilter (String name) {
+ return instance.filters.get(name);
+ }
+
+
+ /**
+ * Get the names of the defined filters.
+ *
+ * @return Returns an array containing the names of filters that
+ * have been registered.
+ *
+ */
+ public static String[] getFilterNames () {
+ synchronized(instance.filters) {
+ String[] filterNames= new String[instance.filters.keySet().size()];
+ instance.filters.keySet().toArray(filterNames);
+ return filterNames;
+ }
+ }
+
+ /**
+ * Internal method which takes care of the job of adding
+ * LogFilter mappings but doesn't perform any of the checks
+ * performed by the public method for adding mappings.
+ */
+ private void addLogFilterInternal (String name, LogFilter filter) {
+ filters.put(name, filter);
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java
new file mode 100644
index 00000000000..fa626de1d08
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.filter;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.event.Count;
+import com.yahoo.log.event.CountGroup;
+import com.yahoo.log.event.Event;
+import com.yahoo.log.event.Histogram;
+import com.yahoo.log.event.MalformedEventException;
+import com.yahoo.log.event.Value;
+import com.yahoo.log.event.ValueGroup;
+import com.yahoo.log.LogMessage;
+
+/**
+ * This filter matches events that are used for monitoring, specificly
+ * the Count and Value events.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class MetricsFilter implements LogFilter {
+ public boolean isLoggable (LogMessage msg) {
+ if (msg.getLevel() != LogLevel.EVENT) {
+ return false;
+ }
+
+ Event event;
+ try {
+ event = msg.getEvent();
+ }
+ catch (MalformedEventException e) {
+ return false;
+ }
+
+ // if it is not Count, Value or something which will generate
+ // Count or Value we don't care
+ if (! ((event instanceof Count)
+ || (event instanceof Value)
+ || (event instanceof Histogram)
+ || (event instanceof CountGroup)
+ || (event instanceof ValueGroup))) {
+ return false;
+ }
+
+ return true;
+ }
+
+ public String description () {
+ return "Match all events representing system metrics (Counts, Values, etc).";
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java
new file mode 100644
index 00000000000..a360cba6b8f
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.filter;
+
+import com.yahoo.log.LogMessage;
+
+/**
+ * Filter which always returns false.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class MuteFilter implements LogFilter {
+ private static final MuteFilter instance = new MuteFilter();
+
+ /**
+ * Singleton, private constructor.
+ */
+ private MuteFilter () {}
+
+ public static MuteFilter getInstance() {
+ return instance;
+ }
+
+ public boolean isLoggable (LogMessage msg) {
+ return false;
+ }
+
+ public String description () {
+ return "Matches no messages. Mute.";
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java
new file mode 100644
index 00000000000..5621ea60d0c
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.filter;
+
+import com.yahoo.log.LogMessage;
+
+/**
+ * This filter is the complement of MetricsFilter
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class NoMetricsFilter implements LogFilter {
+ final MetricsFilter filter = new MetricsFilter();
+
+ public boolean isLoggable (LogMessage msg) {
+ return (! filter.isLoggable(msg));
+ }
+
+ public String description () {
+ return "Matches all log messages except Count and Value events";
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java
new file mode 100644
index 00000000000..a62b2f7171b
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.filter;
+
+import com.yahoo.log.LogMessage;
+
+/**
+ *
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class NullFilter implements LogFilter {
+ public boolean isLoggable (LogMessage msg) {
+ return true;
+ }
+
+ public String description () {
+ return "Match all log messages";
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java
new file mode 100644
index 00000000000..ae217634e1c
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.formatter;
+
+import com.yahoo.log.LogMessage;
+
+/**
+ * This interface is analogous to the java.util.logging.Formatter
+ * interface. Classes implementing this interface should be
+ * <b>stateless/immutable if possible so formatters can be
+ * shared</b>. If it does have state it must not prevent
+ * concurrent use.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public interface LogFormatter {
+ /**
+ * Format log message as a string.
+ *
+ * @param msg The log message
+ *
+ */
+ public String format (LogMessage msg);
+
+ /**
+ * Returns a textual description of the formatter
+ */
+ public String description ();
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java
new file mode 100644
index 00000000000..fb81b35dcf0
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/*
+ * $Id$
+ *
+ */
+
+package com.yahoo.logserver.formatter;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This singleton class implements a central registry of LogFormatter
+ * instances.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class LogFormatterManager {
+ private static final LogFormatterManager instance;
+ static {
+ instance = new LogFormatterManager();
+ instance.addLogFormatterInternal("system.textformatter", new TextFormatter());
+ instance.addLogFormatterInternal("system.nullformatter", new NullFormatter());
+ }
+
+ private final Map<String,LogFormatter> logFormatters =
+ new HashMap<String,LogFormatter>();
+
+ private LogFormatterManager () {}
+
+ /**
+ * LogFormatter lookup function
+ *
+ * @param name The name of the LogFormatter to be looked up.
+ * @return Returns the LogFormatter associated with this name or
+ * <code>null</code> if not found.
+ */
+ public static LogFormatter getLogFormatter (String name) {
+ synchronized(instance.logFormatters) {
+ return instance.logFormatters.get(name);
+ }
+ }
+
+ /**
+ * Get the names of the defined formatters.
+ *
+ * @return Returns an array containing the names of formatters that
+ * have been registered.
+ *
+ */
+ public static String[] getFormatterNames () {
+ synchronized(instance.logFormatters) {
+ String[] formatterNames = new String[instance.logFormatters.keySet().size()];
+ instance.logFormatters.keySet().toArray(formatterNames);
+ return formatterNames;
+ }
+ }
+
+ /**
+ * Internal method which takes care of the job of adding
+ * LogFormatter mappings but doesn't perform any of the checks
+ * performed by the public method for adding mappings.
+ */
+ private void addLogFormatterInternal (String name, LogFormatter logFormatter) {
+ synchronized(logFormatters) {
+ logFormatters.put(name, logFormatter);
+ }
+ }
+
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java
new file mode 100644
index 00000000000..646ab0ce423
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/*
+ * $Id$
+ *
+ */
+
+package com.yahoo.logserver.formatter;
+
+import com.yahoo.log.LogMessage;
+
+/**
+ * This formatter doesn't really format anything. It just
+ * calls the LogMessage toString() method. This is kind of
+ * pointless and silly, but we include it for symmetry...
+ * or completeness....or...whatever.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class NullFormatter implements LogFormatter {
+
+ public String format (LogMessage msg) {
+ return msg.toString();
+ }
+
+ public String description() {
+ return "Format message in native VESPA format";
+ }
+
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java
new file mode 100644
index 00000000000..d94dcceaf29
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/*
+ * $Id$
+ *
+ */
+
+package com.yahoo.logserver.formatter;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+
+import com.yahoo.log.LogMessage;
+
+/**
+ * Creates human-readable text representation of log message.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class TextFormatter implements LogFormatter {
+ static final SimpleDateFormat dateFormat;
+
+ static {
+ dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ }
+
+ public String format (LogMessage msg) {
+ StringBuffer sbuf = new StringBuffer(150);
+ sbuf.append(dateFormat.format(new Date(msg.getTime())))
+ .append(" ")
+ .append(msg.getHost())
+ .append(" ")
+ .append(msg.getThreadProcess())
+ .append(" ")
+ .append(msg.getService())
+ .append(" ")
+ .append(msg.getComponent())
+ .append(" ")
+ .append(msg.getLevel().toString())
+ .append(" ")
+ .append(msg.getPayload())
+ .append("\n");
+
+ return sbuf.toString();
+ }
+
+ public String description() {
+ return "Format log-message as human readable text";
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java
new file mode 100644
index 00000000000..f498568e7f2
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java
@@ -0,0 +1,137 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers;
+
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.filter.LogFilter;
+
+/**
+ * This abstract class is the one you would usually want to
+ * extend when you are writing a LogHandler since it takes care
+ * of quite a bit of tedious work for you (log message counting,
+ * handling of lists of messages versus single instances etc).
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public abstract class AbstractLogHandler implements LogHandler {
+ private long count = 0;
+ private long filtered = 0;
+ private LogFilter filter = null;
+ private String name;
+
+ /**
+ * This is the entry point for each log handler. Takes care
+ * of calling the actual doHandle() method. Provided to make
+ * it possible to extend what a handler does in a uniform way.
+ *
+ * @param msg The message we are about to handle
+ */
+ public final void handle (LogMessage msg) {
+ if ((filter != null) && (! filter.isLoggable(msg))) {
+ filtered++;
+ return;
+ }
+
+ if (doHandle(msg)) {
+ count++;
+ }
+ }
+
+ /**
+ * Handle a list of LogMessage instances
+ *
+ * @param messages List of LogMessage instances.
+ */
+ public final void handle (List<LogMessage> messages) {
+ Iterator<LogMessage> it = messages.iterator();
+ while (it.hasNext()) {
+ handle(it.next());
+ }
+ }
+
+ /**
+ * Set LogFilter for this LogHandler. If the LogFilter is
+ * <code>null</code>, filtering has in effect been turned
+ * off.
+ *
+ * @param filter The filter to be used for this handler
+ *
+ */
+ public void setLogFilter (LogFilter filter) {
+ this.filter = filter;
+ }
+
+ /**
+ * @return Returns the log filter for this handler or
+ * <code>null</code> if no filter is in effect.
+ */
+ public LogFilter getLogFilter () {
+ return filter;
+ }
+
+ /**
+ * Returns the internal counter which keeps track of the number
+ * of times doHandle has been called.
+ *
+ * @return Returns the number of times doHandle has been called.
+ */
+ public final long getCount () {
+ return count;
+ }
+
+ public String getName () {
+ if (name == null) {
+ String n = this.getClass().getName();
+ int x = n.lastIndexOf('.');
+ if (x != -1) {
+ n = n.substring(x+1);
+ }
+ name = n;
+ }
+ return name;
+ }
+
+ public void setName (String name) {
+ this.name = name;
+ }
+
+ /**
+ * The method which actually handles the log message and
+ * does something to it. This is the one you wish to
+ * override when you write a new handler.
+ *
+ * <P>
+ * <em>
+ * If your handle method is slow you should document this fact
+ * so that decisions can be made with regard to configuration.
+ * </em>
+ *
+ * @param msg The LogMessage we are about to handle
+ * @return Returns <code>true</code> if the message was
+ * handled and <code>false</code> if it was ignored.
+ */
+ public abstract boolean doHandle (LogMessage msg);
+
+ /**
+ * Flush LogMessages.
+ */
+ public abstract void flush ();
+
+ /**
+ * Close this loghandler. After a loghandler is closed calling
+ * the #handle() has undefined behavior, but it should be assumed
+ * that log messages will be silently dropped.
+ *
+ * <P>
+ * #close() usually implies #flush() but don't bet on it.
+ */
+ public abstract void close ();
+
+ /**
+ * Force implementation of (hopefully meaningful) toString()
+ */
+ public abstract String toString();
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java
new file mode 100644
index 00000000000..8af5202739b
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java
@@ -0,0 +1,241 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.logging.Logger;
+
+import com.yahoo.io.FatalErrorHandler;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogMessage;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+/**
+ * This handler implements a dispatcher which runs in its own
+ * thread. The purpose of this handler is to isolate execution
+ * of handlers from the main server IO threads.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+
+public class HandlerThread extends Thread implements LogHandler
+{
+ private static final Logger log = Logger.getLogger(HandlerThread.class.getName());
+
+ // default queue size is 200
+ public static final int DEFAULT_QUEUESIZE = 200;
+ private static int queueSize = DEFAULT_QUEUESIZE;
+
+ private FatalErrorHandler fatalErrorHandler;
+
+ // set other queue size if specified
+ static {
+ String queueSizeStr = System.getProperty("logserver.queue.size");
+ if (queueSizeStr != null) {
+ queueSize = Integer.parseInt(queueSizeStr);
+
+ // should never be smaller than 50
+ if (queueSize < 50) {
+ queueSize = 50;
+ }
+
+ log.info("set queue size to " + queueSize);
+ }
+ }
+
+ private static class ItemOrList {
+ final LogMessage item;
+ final List<LogMessage> list;
+ ItemOrList(LogMessage i) {
+ this.item = i;
+ this.list = null;
+ }
+ ItemOrList(List<LogMessage> l) {
+ this.item = null;
+ this.list = l;
+ }
+ public String toString() {
+ return "item="+item+", list="+list;
+ }
+ }
+
+ private final BlockingQueue<ItemOrList> queue;
+ private final List<LogHandler> handlers = new ArrayList<LogHandler>();
+ private long count;
+ @SuppressWarnings("unused")
+ private long droppedCount = 0;
+ @SuppressWarnings("unused")
+ private boolean queueWasFull = false;
+ @SuppressWarnings("unused")
+ private long lastDropLogMessage = 0;
+ @SuppressWarnings("unused")
+ private long lastAcceptingLogMessage = 0;
+
+ public HandlerThread (String name) {
+ super(name);
+ queue = new LinkedBlockingQueue<>(queueSize);
+ log.log(LogLevel.CONFIG, "logserver.queue.size=" + queueSize);
+ }
+
+ /**
+ * Register a handler for fatal errors.
+ *
+ * @param f The FatalErrorHandler instance to be registered
+ */
+ public synchronized void setFatalErrorHandler (FatalErrorHandler f) {
+ fatalErrorHandler = f;
+ }
+
+ /**
+ * Called by the LogDispatch to put a LogMessage onto the Queue
+ *
+ * @param message The LogMessage we wish to dispatch to this
+ * handler thread.
+ */
+ public void handle (LogMessage message) {
+ handleInternal(new ItemOrList(message));
+ }
+
+ /**
+ * Called by the LogDispatch to put a list of LogMessage
+ * instances onto the Queue.
+ */
+ public void handle (List<LogMessage> messages) {
+ handleInternal(new ItemOrList(messages));
+ }
+
+ private void handleInternal (ItemOrList o) {
+ boolean done = false;
+ while (! done) {
+ try {
+ queue.put(o);
+ done = true;
+ }
+ catch (InterruptedException e) {
+ // NOP
+ }
+ }
+ }
+
+ public void flush () {
+ Iterator<LogHandler> it = handlers.iterator();
+ while (it.hasNext()) {
+ LogHandler handler = (LogHandler)it.next();
+ handler.flush();
+ }
+ }
+
+ public void close () {
+ Iterator<LogHandler> it = handlers.iterator();
+ while (it.hasNext()) {
+ LogHandler handler = (LogHandler)it.next();
+ handler.close();
+ }
+ }
+
+ public long getCount () {
+ return count;
+ }
+
+ /**
+ * Register a LogHandler
+ */
+ public synchronized void registerHandler (LogHandler handler) {
+ log.fine("Registering handler " + handler);
+ handlers.add(handler);
+ }
+
+ /**
+ * Unregister a Loghandler
+ */
+ public synchronized void unregisterHandler (LogHandler handler) {
+ int idx;
+ while ((idx = handlers.indexOf(handler)) != -1) {
+ try {
+ handlers.remove(idx);
+ } catch (IndexOutOfBoundsException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Return an array of the registered handlers.
+ *
+ * @return Returns an array of the handlers registered
+ */
+ public LogHandler[] getHandlers () {
+ LogHandler[] h = new LogHandler[handlers.size()];
+ return handlers.toArray(h);
+ }
+
+
+ /**
+ * Return the underlying queue used to send LogMessage instances
+ * to this handler thread.
+ */
+ public BlockingQueue<ItemOrList> getQueue () {
+ return queue;
+ }
+
+ /**
+ * Consume messages from the incoming queue and hand
+ * them off to the handlers.
+ */
+ public void run () {
+ if (queue == null) {
+ throw new NullPointerException("channel is not allowed to be null");
+ }
+
+ // TODO: Make the legmessage elements some kind of composite structure to handle both individual messages and lists uniformly.
+ List<ItemOrList> drainList = new ArrayList<ItemOrList>(queue.size() + 1);
+ try {
+ for (;;) {
+ drainList.clear();
+ // block in take(), then see if there is more
+ // to be had with drainTo()
+ drainList.add(queue.take());
+ queue.drainTo(drainList);
+
+ for (ItemOrList o : drainList) {
+ // we can get two types of elements here: single log
+ // messages or lists of log messages, so we need to
+ // handle them accordingly.
+
+ if (o.item != null) {
+ for (LogHandler handler : handlers) {
+ handler.handle(o.item);
+ }
+ } else if (o.list != null) {
+ for (LogHandler handler : handlers) {
+ handler.handle(o.list);
+ }
+ } else {
+ throw new IllegalArgumentException("not LogMessage or List: " + o);
+ }
+ count++;
+ }
+ }
+ } catch (InterruptedException e) {
+ // NOP
+ } catch (Throwable t) {
+ if (fatalErrorHandler != null) {
+ fatalErrorHandler.handle(t, null);
+ }
+ } finally {
+ log.fine("Handler thread "
+ + getName()
+ + " exiting, removing handlers");
+ for (LogHandler handler : handlers) {
+ log.fine("Removing handler " + handler);
+ handler.close();
+ }
+ handlers.clear();
+ log.fine("Handler thread " + getName() + " done");
+ }
+
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java
new file mode 100644
index 00000000000..6fb009fcda2
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers;
+
+import com.yahoo.log.LogMessage;
+import java.util.List;
+
+/**
+ * The LogHandler interface defines the interface used for all
+ * parts of the logserver which consume log messages.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public interface LogHandler {
+ /**
+ * This is the entry point for the log handling. This method
+ * should return as quickly as possible in implementations
+ * so if you need to initiate time consuming processing you
+ * should look into some design alternatives.
+ *
+ * @param msg The log message
+ */
+ public void handle (LogMessage msg);
+
+ /**
+ * Instead of taking a single log message, this method can take
+ * a List of them. The List abstraction was chosen because the
+ * order needs to be preserved.
+ *
+ * @param messages a List containing zero or more LogMessage
+ * instances.
+ */
+ public void handle (List<LogMessage> messages);
+
+ /**
+ * Any log messages received so far should be dealt with
+ * before this method returns -- within reason ,of course.
+ * (<em>Within reason is loosely defined to be 2-5 seconds</em>)
+ */
+ public void flush ();
+
+ /**
+ * Signals that we want to end logging and should close down the
+ * unerlying logging mechanism -- whatever this maps to
+ * semantically for the underlying implementation. After this
+ * method has been called it is considered an error to submit more
+ * log messages to the handle() methods and an implementation
+ * may elect to throw runtime exceptions.
+ *
+ */
+ public void close ();
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java
new file mode 100644
index 00000000000..93552bf9f00
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java
@@ -0,0 +1,186 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.lasterrorsholder;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.io.Connection;
+import com.yahoo.io.ConnectionFactory;
+import com.yahoo.io.Listener;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.handlers.AbstractLogHandler;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.nio.channels.SocketChannel;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * The LastErrorsHolder handler is used for holding the last n
+ * messages at level error or higher. Connecting to this handler
+ * will return a Json object with the last errors (default is last 100 errors)
+ *
+ * @author musum
+ */
+public class LastErrorsHolder extends AbstractLogHandler implements ConnectionFactory {
+ private static final Logger log = Logger.getLogger(LastErrorsHolder.class.getName());
+ private static final int maxNumErrors = 100;
+ private final Object lock = new Object();
+
+ private int port;
+ private Listener listener;
+
+ private final ArrayList<LogMessage> errors = new ArrayList<>();
+ private int numberOfErrors = 0;
+
+ /**
+ * @param port The port to which this handler listens to.
+ */
+ public LastErrorsHolder(int port) throws IOException {
+ this.port = port;
+ listen(port);
+ }
+
+ public void listen(int port) throws IOException {
+ if (listener != null) {
+ throw new IllegalStateException("already listening to port " + this.port);
+ }
+ listener = new Listener("last-errors-holder");
+ listener.listen(this, port);
+ listener.start();
+ log.log(LogLevel.CONFIG, "port=" + port);
+ }
+
+ public boolean doHandle(LogMessage msg) {
+ if (msg.getLevel().equals(LogLevel.ERROR) || msg.getLevel().equals(LogLevel.FATAL)) {
+ synchronized (lock) {
+ numberOfErrors++;
+ if (errors.size() < maxNumErrors) {
+ errors.add(msg);
+ } else if (numberOfErrors == maxNumErrors) {
+ log.log(LogLevel.DEBUG, String.format("Not storing errors, have reached maximum number of errors: %d, total number of errors received: %d",
+ maxNumErrors, numberOfErrors));
+ }
+ }
+ }
+ return true;
+ }
+
+ public void close() {
+ try {
+ listener.interrupt();
+ listener.join();
+ log.log(LogLevel.DEBUG, "listener stopped");
+ } catch (InterruptedException e) {
+ log.log(LogLevel.WARNING, "listener was interrupted", e);
+ }
+ }
+
+ public void flush() {
+ }
+
+ /**
+ * Factory method for creating new connections. Since we just return a result
+ * when client connection happens, we also write the result here
+ *
+ * @param socket The new SocketChannel
+ * @param listener The Listener instance we want to use
+ */
+ public Connection newConnection(SocketChannel socket, Listener listener) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "New last-errors-holder connection: " + socket);
+ }
+ LastErrorsHolderConnection connection = new LastErrorsHolderConnection(socket);
+ synchronized (lock) {
+ Messages messages = new Messages();
+ for (LogMessage error : errors) {
+ messages.addMessage(
+ new Message(error.getTime()/1000,
+ error.getHost(),
+ error.getService(),
+ error.getLevel().getName(),
+ error.getPayload()));
+ }
+ messages.setNumberOfErrors(numberOfErrors);
+
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ StringWriter stringWriter = new StringWriter();
+ mapper.writeValue(stringWriter, messages);
+ connection.enqueue(StandardCharsets.UTF_8.encode(stringWriter.toString()));
+ } catch (IOException e) {
+ log.log(LogLevel.WARNING, "Could not enqueue log message", e);
+ }
+
+ errors.clear();
+ numberOfErrors = 0;
+ }
+
+ return connection;
+ }
+
+ public String toString() {
+ return LastErrorsHolder.class.getName();
+ }
+
+
+ static class Messages {
+ private final List<Message> messages = new ArrayList<>();
+ private long errorCount = 0; // There might be more errors than number of messages
+
+ void addMessage(Message message) {
+ messages.add(message);
+ }
+
+ void setNumberOfErrors(long errorCount) {
+ this.errorCount = errorCount;
+ }
+
+ public List<Message> getMessages() {
+ return messages;
+ }
+
+ public long getErrorCount() {
+ return errorCount;
+ }
+ }
+
+ static class Message {
+ private final long time;
+ private final String hostname;
+ private final String service;
+ private final String logLevel;
+ private final String message;
+
+ Message(long time, String hostname, String service, String logLevel, String message) {
+ this.time = time;
+ this.hostname = hostname;
+ this.service = service;
+ this.logLevel = logLevel;
+ this.message = message;
+ }
+
+ public long getTime() {
+ return time;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public String getLogLevel() {
+ return logLevel;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public String getService() {
+ return service;
+ }
+ }
+
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java
new file mode 100644
index 00000000000..0dc89e95d11
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.lasterrorsholder;
+
+import com.yahoo.io.Connection;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.filter.LogFilter;
+import com.yahoo.logserver.filter.LogFilterManager;
+import com.yahoo.logserver.formatter.LogFormatter;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+import java.util.logging.Logger;
+
+/**
+ * LastErrorsHandler client connection.
+ *
+ * @author musum
+ */
+public class LastErrorsHolderConnection implements Connection, LogFilter {
+ private static final Logger log = Logger.getLogger(LastErrorsHolderConnection.class.getName());
+
+ private final SocketChannel socket;
+ private ByteBuffer writeBuffer;
+ private final ByteBuffer readBuffer = ByteBuffer.allocate(4096);
+ private LogFilter filter = null;
+ protected LogFormatter formatter = null;
+ private static final String filterName = "system.mute";
+
+ /**
+ * Constructs a LastErrorsHolderConnection. Note that initially the
+ * filter of this connection is set to MuteFilter, which mutes
+ * all log messages.
+ */
+ public LastErrorsHolderConnection(SocketChannel socket) {
+ this.socket = socket;
+ this.filter = LogFilterManager.getLogFilter(filterName);
+ }
+
+ /**
+ * Check if the message is wanted by this particular replicator
+ * connection. The reason we provide this method is because we
+ * want to be able to determine if a message is wanted by any
+ * client before committing resources to creating a ByteBuffer to
+ * serialize it into.
+ *
+ * @param msg The log message offered
+ */
+ public boolean isLoggable(LogMessage msg) {
+ if (filter == null) {
+ return true;
+ }
+ return filter.isLoggable(msg);
+ }
+
+ /**
+ * Return the description of the currently active filter.
+ */
+ public String description() {
+ if (filter == null) {
+ return "No filter defined";
+ }
+ return filter.description();
+ }
+
+
+ /**
+ * Enqueues a ByteBuffer containing the message destined
+ * for the client.
+ *
+ * @param buffer the ByteBuffer into which the log message is
+ * serialized.
+ */
+ public synchronized void enqueue(ByteBuffer buffer) throws IOException {
+ writeBuffer = buffer;
+ write();
+ }
+
+ public void read() throws IOException {
+ if (!readBuffer.hasRemaining()) {
+ log.warning("Log message too long. Message exceeds "
+ + readBuffer.capacity()
+ + " bytes. Connection dropped.");
+ close();
+ return;
+ }
+
+
+ int ret = socket.read(readBuffer);
+ if (ret == -1) {
+ close();
+ return;
+ }
+
+ if (ret == 0) {
+ if (log.isLoggable(LogLevel.INFO)) {
+ log.log(LogLevel.INFO, "zero byte read occurred");
+ }
+ }
+
+ readBuffer.flip();
+ }
+
+ public synchronized void write() throws IOException {
+ if (!socket.isOpen()) {
+ close();
+ }
+ do {
+ try {
+ socket.write(writeBuffer);
+ } catch (IOException e) {
+ log.log(LogLevel.WARNING, "Error writing", e);
+ close();
+ return;
+ }
+ } while (writeBuffer.hasRemaining());
+ }
+
+ public synchronized void close() throws IOException {
+ socket.close();
+ writeBuffer = null;
+ }
+
+ public int selectOps() {
+ return SelectionKey.OP_READ;
+ }
+
+ public SocketChannel socketChannel() {
+ return socket;
+ }
+
+ public void connect() {
+ }
+
+ void setFilter(LogFilter filter) {
+ this.filter = filter;
+ }
+}
+
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java
new file mode 100644
index 00000000000..7cfdbe2b578
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.lasterrorsholder;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.logserver.Server;
+import com.yahoo.plugin.Config;
+import com.yahoo.plugin.Plugin;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+public class LastErrorsHolderPlugin implements Plugin {
+ private static final String DEFAULT_PORT = "19082";
+ private static final Logger log = Logger.getLogger(LastErrorsHolderPlugin.class.getName());
+ private LastErrorsHolder lastErrorsHolder;
+ private final Server server = Server.getInstance();
+
+ public String getPluginName() {
+ return "last-errors-holder";
+ }
+
+ /**
+ * Initialize the plugin
+ */
+ public void initPlugin(Config config) {
+ if (lastErrorsHolder != null) {
+ throw new IllegalStateException("plugin already initialized: " + getPluginName());
+ }
+ int listenPort = config.getInt("port", DEFAULT_PORT);
+ String threadName = config.get("thread", getPluginName());
+ try {
+ lastErrorsHolder = new LastErrorsHolder(listenPort);
+ } catch (IOException e) {
+ log.log(LogLevel.WARNING, "init failed: " + e);
+ return;
+ }
+ server.registerLogHandler(lastErrorsHolder, threadName);
+ }
+
+ /**
+ * Shut down the plugin.
+ */
+ public void shutdownPlugin() {
+
+ if (lastErrorsHolder == null) {
+ throw new IllegalStateException("plugin not initialized: " + getPluginName());
+ }
+ server.unregisterLogHandler(lastErrorsHolder);
+ lastErrorsHolder = null;
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java
new file mode 100644
index 00000000000..fee835b8189
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java
@@ -0,0 +1,233 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/*
+ * $Id$
+ */
+package com.yahoo.logserver.handlers.logmetrics;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.ArrayList;
+import java.util.logging.Logger;
+import java.util.logging.Level;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.event.Event;
+import com.yahoo.logserver.filter.LevelFilter;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.handlers.AbstractLogHandler;
+
+/**
+ * The LogMetricsHandler stores a count of the number of log messages
+ * per level per host and sends an event count for this five minutes.
+ *
+ *
+ * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a>
+ */
+public class LogMetricsHandler extends AbstractLogHandler
+{
+ public static final long EVENTINTERVAL = 5 * 60; // in seconds
+
+ private static final Logger log =
+ Logger.getLogger(LogMetricsHandler.class.getName());
+
+ // A list of log metrics per host and per log level
+ private final List<LevelCount> logMetrics = new ArrayList<LevelCount>();
+
+ // The log levels that are handled by this plugin
+ private static final Level[] levels = {LogLevel.INFO,
+ LogLevel.WARNING,
+ LogLevel.SEVERE,
+ LogLevel.ERROR,
+ LogLevel.FATAL};
+
+
+ /**
+ * Constructor sets a default log filter ignoring the config,
+ * debug and spam levels.
+ */
+ public LogMetricsHandler() {
+ LevelFilter filter = new LevelFilter();
+
+ for (Level level : Arrays.asList(levels)) {
+ filter.addLevel(level);
+ }
+
+ setLogFilter(filter);
+
+ // Start thread that sends events.
+ EventGenerator eventThread = new EventGenerator();
+ new Thread(eventThread).start();
+ }
+
+ public boolean doHandle(LogMessage message) {
+ String host = message.getHost();
+ Level logLevel = message.getLevel();
+
+ boolean found = false;
+ if (logMetrics.size() > 0) {
+ LevelCount count;
+ // Loop through the list logMetrics and check if there
+ // exists an element with the same host and level.
+ for (int i = 0; i < logMetrics.size(); i++) {
+ count = logMetrics.get(i);
+ if (count.getHost().equals(host) &&
+ count.getLevel().getName().equals(logLevel.getName())) {
+ count.addCount(1);
+ found = true;
+ break;
+ }
+ }
+ }
+
+ // There is no element in logMetrics with the same host and
+ // level as in the message, so create a new object and add it
+ // to the list.
+ if (!found) {
+ for (Level level : Arrays.asList(levels)) {
+ LevelCount levelCount;
+ if (level.getName().equals(logLevel.getName())) {
+ levelCount = new LevelCount(host,
+ level,
+ 1);
+ } else {
+ levelCount = new LevelCount(host,
+ level,
+ 0);
+
+ }
+ logMetrics.add(levelCount);
+ }
+ }
+ return true;
+ }
+
+ /**
+ *
+ * Create event count for each log level and report it. For now we
+ * add up the numbers for all host on each level and report that.
+ *
+ */
+ private void sendEvents() {
+ Map<String,Long> levelCount = getMetricsPerLevel();
+ for (Map.Entry<String,Long> entry : levelCount.entrySet()) {
+ String key = entry.getKey();
+ Long count = entry.getValue();
+ Event.count("log_message." + key.toLowerCase(), count.longValue());
+ }
+ }
+
+ public void flush() {}
+
+ public void close() {}
+
+ public String toString() {
+ return LogMetricsHandler.class.getName();
+ }
+
+ /**
+ * Returns the total number of log messages processed by this
+ * plugin.
+ *
+ * @return A count of log messages
+ */
+ public long getMetricsCount() {
+ long count = 0;
+ for (LevelCount levelCount : logMetrics) {
+ count = count + levelCount.getCount();
+ }
+ return count;
+ }
+
+ /**
+ * Returns a Map of log level counts (level is key and count is
+ * value).
+ *
+ * @return A Map of log level counts
+ */
+ public Map<String,Long> getMetricsPerLevel() {
+ Map<String, Long> levelCounts = new TreeMap<String, Long>();
+ // Loop through all levels summing the count for all hosts.
+ for (Level level : Arrays.asList(levels)) {
+ String levelName = level.getName();
+ long count = 0;
+ for (LevelCount levelCount : logMetrics) {
+ if (levelName.equals(levelCount.getLevel().getName())) {
+ count += levelCount.getCount();
+ }
+ }
+ levelCounts.put(levelName, count);
+ }
+ return levelCounts;
+ }
+
+ /**
+ * The LevelCount class represents the number (count) of log
+ * messages with the same log level for a host.
+ *
+ */
+ private class LevelCount {
+ private final String host;
+ private final Level level;
+ private long count;
+
+ LevelCount(String host, Level level, long count) {
+ this.host = host;
+ this.level = level;
+ this.count = count;
+ }
+
+ LevelCount(String host, Level level) {
+ this(host, level, 0);
+ }
+
+ Level getLevel() {
+ return level;
+ }
+
+ String getHost() {
+ return host;
+ }
+
+ long getCount() {
+ return count;
+ }
+
+ void setCount(long count) {
+ this.count = count;
+ }
+
+ void addCount(long add) {
+ count += add;
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Host=" + host + ", level = " + level.getName() +
+ ",count=" + count);
+ return sb.toString();
+ }
+ }
+
+ /**
+ * Implements a thread that sends events every EVENTINTERVAL
+ * seconds.
+ *
+ */
+ private class EventGenerator implements Runnable {
+ public void run() {
+ // Send events every EVENTINTERVAL seconds
+ while (true) {
+ try {
+ Thread.sleep(EVENTINTERVAL * 1000);
+ } catch (InterruptedException e) {
+ log.log(LogLevel.WARNING, e.getMessage());
+ }
+ sendEvents();
+ }
+ }
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java
new file mode 100644
index 00000000000..204e35d52bb
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/*
+ * $Id$
+ */
+package com.yahoo.logserver.handlers.logmetrics;
+
+import java.util.logging.Logger;
+
+import com.yahoo.logserver.Server;
+import com.yahoo.plugin.Config;
+import com.yahoo.plugin.Plugin;
+
+
+public class LogMetricsPlugin implements Plugin
+{
+ private static final Logger log =
+ Logger.getLogger(LogMetricsPlugin.class.getName());
+ private LogMetricsHandler logMetricsHandler;
+ private final Server server = Server.getInstance();
+
+ public String getPluginName() {
+ return "logmetrics";
+ }
+
+ /** Initialize the logmetrics plugin
+ *
+ * @param config Plugin config object, keys used:
+ * <code>thread</code> - name of handler thread this plugin runs in
+ *
+ */
+ public void initPlugin(Config config) {
+ if (logMetricsHandler != null) {
+ log.finer("LogMetricsPlugin doubly initialized");
+ throw new IllegalStateException(
+ "plugin already initialized: " + getPluginName());
+ }
+ String threadName = config.get("thread", getPluginName());
+ logMetricsHandler = new LogMetricsHandler();
+ server.registerLogHandler(logMetricsHandler, threadName);
+ }
+
+ /**
+ * Shut down the logmetrics plugin.
+ */
+ public void shutdownPlugin() {
+ if (logMetricsHandler == null) {
+ log.finer("LogMetricsPlugin shutdown before initialize");
+ throw new IllegalStateException("plugin not initialized: " +
+ getPluginName());
+ }
+ server.unregisterLogHandler(logMetricsHandler);
+ logMetricsHandler = null;
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java
new file mode 100644
index 00000000000..f20adbe3c91
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.replicator;
+
+import java.util.Map;
+import java.util.IdentityHashMap;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.formatter.LogFormatter;
+import com.yahoo.logserver.formatter.LogFormatterManager;
+
+/**
+ * This class is used to cache log messages that have been
+ * formatted into ByteBuffers. The purpose of this class
+ * is to make it easier to support multiple message formats while
+ * still ensuring we don't format more messages than we strictly need
+ * to and that we don't keep around more buffers that we ought to.
+ *
+ * <P>
+ * This is not a general purpose class, I think, so please
+ * refer to the source code of the Replicator class for
+ * information on how to use this.
+ *
+ * <P>
+ * This class is not threadsafe.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class FormattedBufferCache {
+ // the documentation says " All of the methods defined in this
+ // class are safe for use by multiple concurrent threads." so
+ // we have only one instance of the charset for this class.
+ //
+ static private final Charset charset = Charset.forName("utf-8");
+
+ private final IdentityHashMap<LogFormatter, ByteBuffer> buffers;
+
+ public FormattedBufferCache () {
+ // hope this is a good hash size
+ int initialSize = LogFormatterManager.getFormatterNames().length * 2;
+ buffers = new IdentityHashMap<LogFormatter, ByteBuffer>(initialSize);
+ }
+
+ /**
+ * Return a ByteBuffer slice of a buffer containing the
+ * LogMessage formatted by the LogFormatter. If one didn't
+ * exist in the cache from before, it will after this
+ * method returns.
+ *
+ * @param msg The log message you wish to format
+ * @param formatter The log formatter you wish to use for formatting
+ * @return Returns a ByteBuffer slice
+ */
+ public ByteBuffer getFormatted (LogMessage msg, LogFormatter formatter) {
+ ByteBuffer bb = buffers.get(formatter);
+ if (bb == null) {
+ bb = charset.encode(formatter.format(msg));
+ buffers.put(formatter, bb);
+ }
+ return bb.slice();
+ }
+
+ /**
+ * After we're done distributing the log message to all the
+ * clients we clear the cache so we are ready for the next
+ * message.
+ */
+ public void reset () {
+ buffers.clear();
+ }
+
+ /**
+ * This is here for test purposes. Don't get any bright ideas.
+ */
+ public Map<LogFormatter,ByteBuffer> getUnderlyingMapOnlyForTesting() {
+ return buffers;
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java
new file mode 100644
index 00000000000..f8dc700931f
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.replicator;
+
+import java.io.IOException;
+import java.nio.channels.SocketChannel;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Logger;
+import com.yahoo.io.Connection;
+import com.yahoo.io.ConnectionFactory;
+import com.yahoo.io.Listener;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.handlers.AbstractLogHandler;
+
+/**
+ * The Replicator plugin is used for replicating log messages sent
+ * to the logserver.
+ *
+ * <P>
+ * Per default the replicator will start dropping messages enqueued
+ * to a client if the outbound message queue reaches 5000 messages.
+ * This limit can be configured by setting the system property
+ * <code>logserver.replicator.maxqueuelength</code> to the desired
+ * value.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class Replicator extends AbstractLogHandler
+implements ConnectionFactory
+{
+ private static final Logger log = Logger.getLogger(Replicator.class.getName());
+
+ private int port;
+ private Listener listener;
+ private final Set<ReplicatorConnection> connections = new HashSet<ReplicatorConnection>();
+ private final FormattedBufferCache bufferCache = new FormattedBufferCache();
+
+ /**
+ * @param port The port to which the replicator listens.
+ *
+ */
+ public Replicator (int port) throws IOException {
+ this.port = port;
+ listen(port);
+ }
+
+ public Replicator () {
+ }
+
+ public void listen (int port) throws IOException {
+ if (listener != null) {
+ throw new IllegalStateException("already listening to port " + this.port);
+ }
+ listener = new Listener("replicator");
+ listener.listen(this, port);
+ listener.start();
+ log.log(LogLevel.CONFIG, "port=" + port);
+ }
+
+ public synchronized boolean doHandle (LogMessage msg) {
+ boolean logged = false;
+ bufferCache.reset();
+ for (ReplicatorConnection c : connections) {
+ try {
+ if (c.isLoggable(msg)) {
+ c.enqueue(bufferCache.getFormatted(msg, c.formatter));
+ logged = true;
+ }
+ }
+ catch (IOException e) {
+ log.log(LogLevel.DEBUG, "Writing failed", e);
+ }
+ }
+ return logged;
+ }
+
+ public void close() {
+ // kill the listener thread, then wait for it to
+ // shut down.
+ try {
+ listener.interrupt();
+ listener.join();
+ log.log(LogLevel.DEBUG, "Replicator listener stopped");
+ }
+ catch (InterruptedException e) {
+ log.log(LogLevel.WARNING,
+ "Replicator listener was interrupted",
+ e);
+ }
+ }
+
+ /**
+ * Currently a NOP, but we might want to have some best-effort
+ * mechanism for trying to flush all connections within some
+ * time-frame.
+ */
+ public void flush() {}
+
+ /**
+ * Factory method for wrapping new connections in the proper
+ * (Replicator)Connection objects.
+ *
+ * @param socket The new SocketChannel
+ * @param listener The Listener instance we want to use
+ */
+ public synchronized Connection newConnection (SocketChannel socket,
+ Listener listener) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.fine("New replicator connection: " + socket);
+ }
+ ReplicatorConnection n =
+ new ReplicatorConnection(socket, listener, this);
+ connections.add(n);
+ return n;
+ }
+
+ /**
+ * Removes a ReplicatorConnection from the set of active
+ * connections.
+ */
+ protected synchronized void deRegisterConnection (ReplicatorConnection conn) {
+ connections.remove(conn);
+ }
+
+ public String toString () {
+ return Replicator.class.getName();
+ }
+
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java
new file mode 100644
index 00000000000..2978f751665
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java
@@ -0,0 +1,411 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.replicator;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+import java.util.LinkedList;
+import java.util.StringTokenizer;
+import java.util.logging.Logger;
+
+import com.yahoo.io.Connection;
+import com.yahoo.io.IOUtils;
+import com.yahoo.io.Listener;
+import com.yahoo.io.ReadLine;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.filter.LogFilter;
+import com.yahoo.logserver.filter.LogFilterManager;
+import com.yahoo.logserver.formatter.LogFormatter;
+import com.yahoo.logserver.formatter.LogFormatterManager;
+
+/**
+ * Replication client connection.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class ReplicatorConnection implements Connection, LogFilter
+{
+ private static final Logger log
+ = Logger.getLogger(ReplicatorConnection.class.getName());
+
+ /** The maximum number of queued messages before we start dropping */
+ private static final int maxQueueLength;
+ /** The maximum number of times we go over maxQueueLength before we log a warning */
+ private static final int maxRetriesBeforeWarning = 10;
+ /** Count of how many times we have received a message while the queue is full */
+ private static int queueFullCount = 0;
+
+ static {
+ String maxQueue = System.getProperty("logserver.replicator.maxqueuelength",
+ "5000");
+ maxQueueLength = Integer.parseInt(maxQueue);
+ }
+
+ private final SocketChannel socket;
+ private final String remoteHost;
+ private final Listener listener;
+ private final Replicator replicator;
+ private final LinkedList<ByteBuffer> writeBufferList = new LinkedList<ByteBuffer>();
+ private ByteBuffer writeBuffer;
+ private final ByteBuffer readBuffer = ByteBuffer.allocate(4096);
+ private LogFilter filter = null;
+ protected LogFormatter formatter = null;
+ private String filterName = "system.mute";
+ private String formatterName = "system.nullformatter";
+ private boolean droppingMode = false;
+ private int numHandled = 0;
+ private int numQueued = 0;
+ private int numDropped = 0;
+ private long totalBytesWritten = 0;
+
+ /**
+ * Constructs a ReplicatorConnection. Note that initially the
+ * filter of this connection is set to MuteFilter, which mutes
+ * all log messages.
+ *
+ */
+ public ReplicatorConnection (SocketChannel socket,
+ Listener listener,
+ Replicator replicator) {
+ this.socket = socket;
+ this.listener = listener;
+ this.replicator = replicator;
+ this.filter = LogFilterManager.getLogFilter(filterName);
+ this.formatter = LogFormatterManager.getLogFormatter(formatterName);
+
+ // this might take some time
+ remoteHost = socket.socket().getInetAddress().getHostName();
+
+ }
+
+ /**
+ * Returns the remote hostname of this replicator connection.
+ *
+ * @return Returns the remote host name
+ */
+ public String getRemoteHost() {
+ return remoteHost;
+ }
+
+ /**
+ * Check if the message is wanted by this particular replicator
+ * connection. The reason we provide this method is because we
+ * want to be able to determine of a message is wanted by any
+ * client before committing resources to creating a ByteBuffer to
+ * serialize it into.
+ *
+ * @param msg The log message offered
+ *
+ */
+ public boolean isLoggable (LogMessage msg) {
+ if (filter == null) {
+ return true;
+ }
+ return filter.isLoggable(msg);
+ }
+
+ /**
+ * Return the description of the currently active filter.
+ */
+ public String description () {
+ if (filter == null) {
+ return "No filter defined";
+ }
+ return filter.description();
+ }
+
+
+ /**
+ * Enqueues a ByteBuffer containing the message destined
+ * for the client.
+ *
+ * @param buffer the ByteBuffer into which the log message is
+ * serialized.
+ */
+ public synchronized void enqueue (ByteBuffer buffer) throws IOException {
+ if (writeBuffer == null) {
+ writeBuffer = buffer;
+ } else {
+ // if we've reached the max we bail out
+ if (writeBufferList.size() > maxQueueLength) {
+ queueFullCount++;
+ if (! droppingMode) {
+ droppingMode = true;
+ String message = "client at " + remoteHost + " can't keep up, dropping messages";
+ if (queueFullCount > maxRetriesBeforeWarning) {
+ log.log(LogLevel.WARNING, message);
+ queueFullCount = 0;
+ } else {
+ log.log(LogLevel.DEBUG, message);
+ }
+ }
+ numDropped++;
+ return;
+ }
+ writeBufferList.addLast(buffer);
+ listener.modifyInterestOps(this, SelectionKey.OP_WRITE, true);
+ droppingMode = false;
+ numQueued++;
+ }
+ numHandled++;
+ write();
+ }
+
+
+ public void read () throws IOException {
+ if (! readBuffer.hasRemaining()) {
+ log.warning("Log message too long. Message exceeds "
+ + readBuffer.capacity()
+ + " bytes. Connection dropped.");
+ close();
+ return;
+ }
+
+
+ int ret = socket.read(readBuffer);
+ if (ret == -1) {
+ close();
+ return;
+ }
+
+ if (ret == 0) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.fine("zero byte read occurred");
+ }
+ }
+
+ readBuffer.flip();
+
+ String s;
+ while ((s = ReadLine.readLine(readBuffer)) != null) {
+ onCommand(s);
+ }
+
+ }
+
+ public synchronized void write () throws IOException {
+ if (! socket.isOpen()) {
+ // throw new IllegalStateException("SocketChannel not open in write()");
+ close();
+ }
+
+ int bytesWritten;
+ do {
+ // if writeBufferList is not set we need to fetch the next buffer
+ if (writeBuffer == null) {
+
+ // if the list is empty, signal the selector we do not need
+ // to do any writing for a while yet and bail
+ if (writeBufferList.isEmpty()) {
+ listener.modifyInterestOpsBatch(this,
+ SelectionKey.OP_WRITE,
+ false);
+ return;
+ }
+ writeBuffer = writeBufferList.removeFirst();
+ }
+
+
+ // invariants: we have a writeBuffer
+
+ // when the client drops off we actually need
+ // to handle that here and close the connection
+ // XXX: I am not sure why this works and the
+ // close method call on IOException in
+ // Listener doesn't. this should be investigated!
+ //
+ try {
+ bytesWritten = socket.write(writeBuffer);
+ }
+ catch (IOException e) {
+ close();
+ return;
+ }
+ totalBytesWritten += bytesWritten;
+
+ // buffer drained so we forget it and see what happens when we
+ // go around. if indeed we go around
+ if ((writeBuffer != null) && (!writeBuffer.hasRemaining())) {
+ writeBuffer = null;
+ }
+ } while (bytesWritten > 0);
+ }
+
+ public synchronized void close () throws IOException {
+ replicator.deRegisterConnection(this);
+ socket.close();
+ writeBuffer = null;
+ writeBufferList.clear();
+ log.log(LogLevel.DEBUG, "closing connection to " + remoteHost);
+ }
+
+ public int selectOps () {
+ return SelectionKey.OP_READ;
+ }
+
+ public SocketChannel socketChannel () {
+ return socket;
+ }
+
+ public void connect () {
+ }
+
+
+ // ========================================================
+ // ==== command processing
+ // ========================================================
+
+ void onCommand (String s) {
+ log.log(LogLevel.DEBUG, "COMMAND: '" + s + "' from " + remoteHost);
+ StringTokenizer st = new StringTokenizer(s.toLowerCase());
+ while (st.hasMoreTokens()) {
+ String tok = st.nextToken();
+ if ("ping".equals(tok)) {
+ if (st.hasMoreTokens()) {
+ print("# 202 pong " + st.nextToken() + "\n");
+ } else {
+ print("# 202 pong\n");
+ }
+ return;
+ }
+
+ if ("use".equals(tok)) {
+ if (st.hasMoreTokens()) {
+ onUse(st.nextToken());
+ }
+ return;
+ }
+
+ if ("formatter".equals(tok)) {
+ if (st.hasMoreTokens()) {
+ onFormatter(st.nextToken());
+ }
+ return;
+ }
+
+ if ("quit".equals(tok)) {
+ print("# 201 bye\n");
+ try { close(); } catch (IOException e) {e.printStackTrace();}
+ return;
+ }
+
+ if ("list".equals(tok)) {
+ onList();
+ return;
+ }
+
+ if ("listformatters".equals(tok)) {
+ onListFormatters();
+ return;
+ }
+
+ if ("stats".equals(tok)) {
+ onStats();
+ return;
+ }
+ }
+ }
+
+ void onFormatter (String formatterName) {
+ LogFormatter newFormatter = LogFormatterManager.getLogFormatter(formatterName);
+ if (newFormatter == null) {
+ print("# 405 formatter not found '" + formatterName + "'\n");
+ return;
+ }
+ formatter = newFormatter;
+ this.formatterName = formatterName;
+ print("# 202 using '" + formatter + "'\n");
+ }
+
+ void onUse (String filterName) {
+ LogFilter newFilter = LogFilterManager.getLogFilter(filterName);
+ if (newFilter == null) {
+ print("# 404 filter not found '" + filterName + "'\n");
+ return;
+ }
+ filter = newFilter;
+ this.filterName = filterName;
+ print("# 200 using '" + filter + "'\n");
+ }
+
+
+
+ void onList () {
+ print("# 203 filter list\n");
+ String filterNames[] = LogFilterManager.getFilterNames();
+ for (int i = 0; i < filterNames.length; i++) {
+ LogFilter f = LogFilterManager.getLogFilter(filterNames[i]);
+ print("# 204 " + filterNames[i] + " - " + f.description() + "\n");
+ }
+ print("# 205 end filter list\n");
+ }
+
+ void onListFormatters () {
+ print("# 206 formatter list\n");
+ String formatterNames[] = LogFormatterManager.getFormatterNames();
+ for (int i = 0; i < formatterNames.length; i++) {
+ LogFormatter fmt = LogFormatterManager.getLogFormatter(formatterNames[i]);
+ print("# 207 " + formatterNames[i] + " - " + fmt.description() + "\n");
+ }
+ print("# 208 end formatter list\n");
+ }
+
+ private void print (String s) {
+ try {
+ enqueue(IOUtils.utf8ByteBuffer(s));
+ }
+ catch (IOException e) {
+ log.log(LogLevel.WARNING, "error printing", e);
+ try {
+ close();
+ }
+ catch (IOException e2) {
+ // ignore
+ }
+ }
+ }
+
+ void onStats () {
+
+ print(new StringBuilder(80)
+ .append("# 206 stats start (this connection)\n")
+ .append("# 207 ").append(numHandled).append(" handled\n")
+ .append("# 208 ").append(numDropped).append(" dropped\n")
+ .append("# 209 ").append(numQueued)
+ .append(" handled and queued\n")
+ .append("# 210 ").append(totalBytesWritten)
+ .append(" total bytes written\n")
+ .append("# 211 stats end\n")
+ .toString()
+ );
+ }
+
+
+ public int getNumHandled () {
+ return numHandled;
+ }
+
+ public int getNumQueued () {
+ return numQueued;
+ }
+
+ public int getNumDropped () {
+ return numDropped;
+ }
+
+ public long getTotalBytesWritten () {
+ return totalBytesWritten;
+ }
+
+ public String getLogFilterName () {
+ return filterName;
+ }
+
+ void setFilter(LogFilter filter) {
+ this.filter = filter;
+ }
+
+}
+
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java
new file mode 100644
index 00000000000..03bc66c8262
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.replicator;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.logserver.Server;
+import com.yahoo.plugin.Config;
+import com.yahoo.plugin.Plugin;
+
+public class ReplicatorPlugin implements Plugin
+{
+ private static final String DEFAULT_PORT = "19083";
+ private static final Logger log =
+ Logger.getLogger(ReplicatorPlugin.class.getName());
+ private Replicator replicator;
+ private final Server server = Server.getInstance();
+
+ public String getPluginName () {
+ return "replicator";
+ }
+
+ /**
+ * Initialize the replicator plugin
+ */
+ public void initPlugin (Config config) {
+
+ if (replicator != null) {
+ throw new IllegalStateException(
+ "plugin already initialized: " + getPluginName());
+ }
+ int listenPort = config.getInt("port", DEFAULT_PORT);
+ String threadName = config.get("thread", getPluginName());
+ try {
+ replicator = new Replicator(listenPort);
+ }
+ catch (IOException e) {
+ log.log(LogLevel.WARNING, "init failed: " + e);
+ return;
+ }
+ server.registerLogHandler(replicator, threadName);
+ }
+
+ /**
+ * Shut down the replicator plugin.
+ */
+ public void shutdownPlugin () {
+
+ if (replicator == null) {
+ throw new IllegalStateException(
+ "plugin not initialized: " + getPluginName());
+ }
+ server.unregisterLogHandler(replicator);
+ replicator = null;
+ }
+
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java b/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java
new file mode 100644
index 00000000000..1a48c5fb0f4
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java
@@ -0,0 +1,272 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.net;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.logserver.LogDispatcher;
+import com.yahoo.log.LogMessage;
+import com.yahoo.log.InvalidLogFormatException;
+
+import com.yahoo.io.Connection;
+import com.yahoo.io.Listener;
+import com.yahoo.io.ReadLine;
+
+import com.yahoo.logserver.net.control.Levels;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+import java.util.logging.Level;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.HashSet;
+
+import java.nio.charset.Charset;
+import java.nio.ByteBuffer;
+import java.nio.channels.SocketChannel;
+import java.nio.channels.SelectionKey;
+
+/**
+ * TODO
+ * <UL>
+ * <LI> send invalid log messages to somewhere so they can be
+ * analyzed and errors can be corrected.
+ * </UL>
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+
+public class LogConnection implements Connection {
+ private static final Logger log = Logger.getLogger(LogConnection.class.getName());
+
+ public static final int READBUFFER_SIZE = (32 * 1024);
+
+ static private final Charset charset = Charset.forName("utf-8");
+
+ // the set of active connections
+ private static final Set<LogConnection> activeConnections = new HashSet<>();
+
+ private final SocketChannel socket;
+ private final Listener listener;
+ private final LogDispatcher dispatcher;
+
+ private final ByteBuffer readBuffer = ByteBuffer.allocateDirect(READBUFFER_SIZE);
+
+ private final LinkedList<ByteBuffer> writeBufferList = new LinkedList<>();
+ private ByteBuffer writeBuffer;
+
+ // counters
+ private long totalBytesRead = 0;
+ private long totalBytesWritten = 0;
+
+ // default log levels for logd
+ final Levels defaultLevels;
+
+ public LogConnection (SocketChannel socket,
+ Listener listener,
+ LogDispatcher dispatcher,
+ Levels defaultLevels) {
+ this.socket = socket;
+ this.listener = listener;
+ this.dispatcher = dispatcher;
+ this.defaultLevels = defaultLevels;
+
+ addToActiveSet(this);
+
+ // send the "setdefaultstate" command to logd. no better
+ // place to put it for now.
+ sendDefaultState();
+ }
+
+ /**
+ * Send the default state to the
+ */
+ public void sendDefaultState () {
+ if (defaultLevels == null) {
+ return;
+ }
+
+ try {
+ enqueue(charset.encode("setdefaultstate "
+ + defaultLevels.toString()
+ + "\n"));
+ enqueue(charset.encode("setallstates "
+ + defaultLevels.toString()
+ + "\n"));
+ }
+ catch (IOException e) {
+ log.log(LogLevel.WARNING, "Unable to send default state", e);
+ }
+ }
+
+ /**
+ * Return a shallow copy of the set of active connections.
+ *
+ */
+ public static Set<LogConnection> getActiveConnections () {
+ synchronized(activeConnections) {
+ return new HashSet<>(activeConnections);
+ }
+ }
+
+ /**
+ * @return Return total number of bytes written to connection
+ */
+ public long getTotalBytesWritten () {
+ return totalBytesWritten;
+ }
+
+ /**
+ * @return Return total number of bytes read from connection
+ */
+ public long getTotalBytesRead () {
+ return totalBytesRead;
+ }
+
+ /**
+ * Internal method for adding connection to the set
+ * of active connections.
+ *
+ * @param connection The connection to be added
+ */
+ private static void addToActiveSet (LogConnection connection) {
+ synchronized(activeConnections) {
+ activeConnections.add(connection);
+ }
+ }
+
+ /**
+ * Internal method to remove connection from the set of
+ * active connections.
+ *
+ * @param connection The connection to remove
+ * @throws IllegalStateException if the connection does not
+ * exist in the set
+ *
+ */
+ private static void removeFromActiveSet (LogConnection connection) {
+ synchronized(activeConnections) {
+ activeConnections.remove(connection);
+ }
+ }
+
+ /**
+ *
+ */
+ public synchronized void enqueue (ByteBuffer buffer) throws IOException {
+ if (writeBuffer == null) {
+ writeBuffer = buffer;
+ } else {
+ writeBufferList.addLast(buffer);
+ listener.modifyInterestOps(this, SelectionKey.OP_WRITE, true);
+ }
+ write();
+ }
+
+ public void connect () throws IOException {
+ throw new RuntimeException("connect() is not supposed to be called");
+ }
+
+ public synchronized void write () throws IOException {
+ int bytesWritten;
+ do {
+ // if writeBufferList is not set we need to fetch the next buffer
+ if (writeBuffer == null) {
+
+ // if the list is empty, signal the selector we do not need
+ // to do any writing for a while yet and bail
+ if (writeBufferList.isEmpty()) {
+ listener.modifyInterestOpsBatch(this,
+ SelectionKey.OP_WRITE,
+ false);
+ return;
+ }
+ writeBuffer = writeBufferList.removeFirst();
+ }
+
+ // invariants: we have a writeBuffer
+
+ bytesWritten = socket.write(writeBuffer);
+ totalBytesWritten += bytesWritten;
+
+ // buffer drained so we forget it and see what happens when we
+ // go around. if indeed we go around
+ if ((writeBuffer != null) && (!writeBuffer.hasRemaining())) {
+ writeBuffer = null;
+ }
+ } while (bytesWritten > 0);
+ }
+
+ public void read() throws IOException {
+ if (! readBuffer.hasRemaining()) {
+
+ try {
+ readBuffer.putChar(readBuffer.capacity() - 2, '\n');
+ readBuffer.flip();
+ String s = ReadLine.readLine(readBuffer);
+ if (s == null) {
+ return;
+ }
+ int count = 200;
+ log.log(LogLevel.WARNING, "Log message too long. Message from "
+ + socket.socket().getInetAddress() + " exceeds "
+ + readBuffer.capacity()
+ + ". Skipping buffer (might be part of same long message). Printing first " + count + " characters of line: " +
+ s.substring(0, count));
+
+ LogMessage msg = LogMessage.parseNativeFormat(s);
+ dispatcher.handle(msg);
+ }
+ catch (InvalidLogFormatException e) {
+ log.log(LogLevel.DEBUG, "Invalid log message", e);
+ }
+ finally {
+ readBuffer.clear();
+ }
+ return;
+ }
+
+ int ret = socket.read(readBuffer);
+ if (ret == -1) {
+ close();
+ return;
+ }
+
+ if (ret == 0) {
+ if (log.isLoggable(Level.FINE)) {
+ log.log(LogLevel.DEBUG, "zero byte read occurred");
+ }
+ }
+
+ // update global counter
+ totalBytesRead += ret;
+
+ readBuffer.flip();
+
+ String s;
+ while ((s = ReadLine.readLine(readBuffer)) != null) {
+ try {
+ LogMessage msg = LogMessage.parseNativeFormat(s);
+ dispatcher.handle(msg);
+ }
+ catch (InvalidLogFormatException e) {
+ log.log(LogLevel.DEBUG, "Invalid log message", e);
+ }
+ }
+ }
+
+ public void close() throws IOException {
+ if (log.isLoggable(Level.FINE)) {
+ log.log(LogLevel.INFO, this + ": closing");
+ }
+ socket.close();
+ removeFromActiveSet(this);
+ }
+
+ public int selectOps() {
+ return SelectionKey.OP_READ;
+ }
+
+ public SocketChannel socketChannel() {
+ return socket;
+ }
+
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java b/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java
new file mode 100644
index 00000000000..5d97420f30a
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.net;
+
+import com.yahoo.logserver.LogDispatcher;
+import com.yahoo.io.Connection;
+import com.yahoo.io.Listener;
+import com.yahoo.io.ConnectionFactory;
+
+import com.yahoo.logserver.net.control.Levels;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.nio.channels.SocketChannel;
+
+/**
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+
+public class LogConnectionFactory implements ConnectionFactory
+{
+ private static final Logger log
+ = Logger.getLogger(LogConnectionFactory.class.getName());
+
+ final LogDispatcher dispatcher;
+ final Levels defaultLogLevels;
+
+ public LogConnectionFactory (LogDispatcher dispatcher) {
+ this.dispatcher = dispatcher;
+ defaultLogLevels = Levels.parse(System.getProperty("logserver.default.loglevels", ""));
+ }
+
+ public Connection newConnection (SocketChannel socket, Listener listener) {
+ if (log.isLoggable(Level.FINE)) {
+ log.fine("New connection: " + socket);
+ }
+ return new LogConnection(socket,
+ listener,
+ dispatcher,
+ (Levels)defaultLogLevels.clone());
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java b/logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java
new file mode 100644
index 00000000000..c9b749f4417
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java
@@ -0,0 +1,125 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.net.control;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * This class is used to represent the state of each log level
+ * in a set of states.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class Levels implements Cloneable {
+ private final Map<String, State> levelsMap = new LinkedHashMap<String, State>(10);
+
+ /**
+ * The constructor initializes the Levels object to its default
+ * state.
+ */
+ public Levels () {
+ levelsMap.put("event", State.FORWARD);
+ levelsMap.put("fatal", State.FORWARD);
+ levelsMap.put("error", State.FORWARD);
+ levelsMap.put("warning", State.FORWARD);
+ levelsMap.put("info", State.FORWARD);
+ levelsMap.put("config", State.FORWARD);
+ levelsMap.put("debug", State.NOFORWARD);
+ levelsMap.put("spam", State.NOFORWARD);
+ }
+
+ /**
+ * Parse a levels representation and return a Levels object
+ * representing the levels.
+ *
+ * @param levels A string representation of the levels
+ * @return new instance of Levels, possibly having no
+ * real values if none could be found in the
+ * <code>levels</code> parameter.
+ *
+ */
+ public static Levels parse (String levels) {
+ return (new Levels()).updateLevels(levels);
+ }
+
+
+ /**
+ * Update the levels given a string representation of the state;
+ * the levels mentioned here will be updated, the ones omitted
+ * will retain their state as before the function call.
+ *
+ * @param levels string representation of levels
+ *
+ */
+ public Levels updateLevels (String levels) {
+ String[] parts = levels.split(",");
+ if (parts.length < 1) {
+ return this;
+ }
+
+ for (int i = 0; i < parts.length; i++) {
+ String pair = parts[i];
+ int offset = pair.indexOf('=');
+ if (offset != -1) {
+ String name = pair.substring(0,offset).trim().toLowerCase();
+ String value = pair.substring(offset+1).trim().toLowerCase();
+ setLevelState(name, State.parse(value));
+ }
+ }
+ return this;
+ }
+
+
+ /**
+ * Set the state of a given level.
+ *
+ * @param level name of the level
+ * @param state the state
+ * @return returns reference to <code>this</code> for chaning
+ */
+ public Levels setLevelState(String level, State state) {
+ levelsMap.put(level, state);
+ return this;
+ }
+
+ /**
+ * Get the state of a given level.
+ *
+ */
+ public State getLevelState (String level) {
+ State s = levelsMap.get(level);
+ if (s == null) {
+ return State.UNKNOWN;
+ }
+ return s;
+ }
+
+ /**
+ * For equivalent configurations the toString method should
+ * emit equal strings.
+ *
+ */
+ public String toString () {
+ StringBuilder sbuf = new StringBuilder(80);
+ boolean first = true;
+ for (Map.Entry<String, State> me : levelsMap.entrySet()) {
+ // commas between
+ if (!first) {
+ sbuf.append(',');
+ } else {
+ first = false;
+ }
+ sbuf.append(me.getKey())
+ .append('=')
+ .append(me.getValue());
+ }
+
+ return sbuf.toString();
+ }
+
+ public Object clone() {
+ // quick and dirty, but easily verifiable to be correct
+ return parse(this.toString());
+ }
+
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/net/control/State.java b/logserver/src/main/java/com/yahoo/logserver/net/control/State.java
new file mode 100644
index 00000000000..5d2a4f0fbd4
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/net/control/State.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.net.control;
+
+import java.util.Map;
+import java.util.HashMap;
+
+/**
+ * This value class represents the logging state of a component.
+ * the valid states are:
+ *
+ * <UL>
+ * <LI> forward - store locally and send to log server
+ * <LI> store - store locally only
+ * <LI> noforward - do not send to logserver
+ * <LI> off - do not generate the message in the program
+ * </UL>
+ *
+ * XXX This does not appear to be in use.
+ */
+public class State {
+ private static final Map<String, State> nameToState = new HashMap<String, State>();
+
+ public static final State FORWARD = new State("forward");
+ public static final State NOFORWARD = new State("noforward");
+ // public static final State STORE = new State("store");
+ // public static final State OFF = new State("off");
+ public static final State UNKNOWN = new State("unknown");
+
+ private String name;
+
+ /**
+ * Typesafe enum. Only able to instantiate self.
+ * TODO: Rewrite to enum
+ */
+ private State () {}
+
+ /**
+ * Creates state with given name
+ */
+ private State (String name) {
+ this.name = name;
+ synchronized (State.class) {
+ nameToState.put(name, this);
+ }
+ }
+
+ public static State parse (String s) {
+ return nameToState.containsKey(s) ? nameToState.get(s) : UNKNOWN;
+ }
+
+ public String toString () {
+ return name;
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java
new file mode 100644
index 00000000000..623bf04aa36
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.testutils;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.event.MalformedEventException;
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogMessage;
+
+/**
+ * This utility is used to check that the log messages contained
+ * in a log file are correct. Any incorrectly formatted log
+ * message is output to stdout.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class VerifyLogfile {
+
+ public static void main (String[] args) throws IOException {
+ int messages = 0;
+ int events = 0;
+ int invalidLogMessages = 0;
+ int invalidEvents = 0;
+ int numFiles = 0;
+
+ if (args.length < 1) {
+ System.err.println("\nPlease provide name of log file(s)\n");
+ }
+
+ for (int i = 0; i < args.length; i++) {
+ BufferedReader br = new BufferedReader(new FileReader(args[i]));
+ numFiles++;
+ for (String line = br.readLine();
+ line != null;
+ line = br.readLine())
+ {
+ messages++;
+ LogMessage m;
+ try {
+ m = LogMessage.parseNativeFormat(line);
+ if (m.getLevel() == LogLevel.EVENT) {
+ events++;
+ m.getEvent();
+ }
+ } catch (MalformedEventException e) {
+ System.out.println("EVENT\t" + line);
+ invalidEvents++;
+ } catch (InvalidLogFormatException e) {
+ System.out.println("MESSAGE\t" + line);
+ invalidLogMessages++;
+ }
+ }
+ br.close();
+ }
+
+ System.err.println("numFiles: " + numFiles);
+ System.err.println("messages: " + messages);
+ System.err.println("events: " + events);
+ System.err.println("invalidLogMessages: " + invalidLogMessages);
+ System.err.println("invalidEvents: " + invalidEvents);
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/plugin/Config.java b/logserver/src/main/java/com/yahoo/plugin/Config.java
new file mode 100644
index 00000000000..a7ea2b44caf
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/plugin/Config.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.plugin;
+
+/**
+ * This interface specifies an API for configuring runtime-loadable
+ * server plugins.
+ *
+ * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a>
+ */
+public abstract class Config
+{
+ /**
+ * @return a config value for the specified key
+ */
+ public abstract String get(String key, String defaultValue);
+
+ /**
+ * @return a config value as an integer
+ */
+ public int getInt(String key, String defaultValue) {
+ return Integer.parseInt(get(key, defaultValue));
+ }
+}
diff --git a/logserver/src/main/java/com/yahoo/plugin/Plugin.java b/logserver/src/main/java/com/yahoo/plugin/Plugin.java
new file mode 100644
index 00000000000..e53bc3a1faa
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/plugin/Plugin.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.plugin;
+
+/**
+ * This interface specifies an API for runtime-loadable server
+ * plugins. The interface is deliberately simple to allow it to be
+ * used in different servers. Typically, the initPlugin() method
+ * calls application-specific registration methods to connect the
+ * plugin to the hosting application.
+ *
+ * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a>
+ */
+public interface Plugin
+{
+ /**
+ * @return a unique and simple name for the plugin
+ */
+ public String getPluginName();
+
+ /**
+ * Initialize the plugin.
+ *
+ * @param config Config object for this plugin
+ */
+ public void initPlugin(Config config);
+
+ /**
+ * Shut down the plugin. Must clean up all resources allocated by
+ * initPlugin() or any of the handler methods.
+ */
+ public void shutdownPlugin();
+
+}
diff --git a/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java b/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java
new file mode 100644
index 00000000000..763dc2d1f3e
--- /dev/null
+++ b/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.plugin;
+
+/**
+ * This class implements plugin config through system properties.
+ * Each plugin typically has its own system property prefix, such as
+ * "logserver.archiver.". A request for the config key "foo" will
+ * then return the contents of the "logserver.archiver.foo" system
+ * property.
+ *
+ * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a>
+ */
+public class SystemPropertyConfig extends Config
+{
+ private final String prefix;
+
+ /**
+ * @param prefix Prefix string prepended to config keys
+ * as they are looked up as system properties.
+ */
+ public SystemPropertyConfig(String prefix) {
+ this.prefix = prefix;
+ }
+
+ /**
+ * @return a config value for the specified key
+ */
+ public String get(String key, String defaultValue) {
+ return System.getProperty(prefix + key, defaultValue);
+ }
+
+ public String toString () {
+ return "Prefix=" + prefix;
+ }
+}
diff --git a/logserver/src/test/files/event.txt.gz b/logserver/src/test/files/event.txt.gz
new file mode 100644
index 00000000000..9422ca5fc90
--- /dev/null
+++ b/logserver/src/test/files/event.txt.gz
Binary files differ
diff --git a/logserver/src/test/files/logEntries.txt b/logserver/src/test/files/logEntries.txt
new file mode 100644
index 00000000000..776e3de882d
--- /dev/null
+++ b/logserver/src/test/files/logEntries.txt
@@ -0,0 +1,3 @@
+1096639280.524133 malfunction 26851 - logtest info Starting up, called as ./log/logtest
+1096639280.524164 malfunction 26851 - logtest info backslash: \\
+1096639280.524133 malfunction 26851 - logtest fatal Starting up, called as ./log/logtest
diff --git a/logserver/src/test/files/value-events.txt b/logserver/src/test/files/value-events.txt
new file mode 100644
index 00000000000..20143d9cd53
--- /dev/null
+++ b/logserver/src/test/files/value-events.txt
@@ -0,0 +1,982 @@
+1107248423.987624 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248423.987754 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248423.987809 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248439.235755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248439.235871 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248439.236632 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248439.236684 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248423.987624 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248423.987754 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248423.987809 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248439.235755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248439.235871 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248439.236632 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248439.236684 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248439.345800 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248439.345924 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248454.346566 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248454.346701 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248454.346755 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248484.708180 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248484.708309 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248484.708362 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248499.377732 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248499.377855 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248499.378618 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248499.378670 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248499.379345 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248499.379407 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248515.068667 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248515.068795 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248515.068848 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248545.429750 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248545.429879 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248545.429933 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248559.509728 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248559.509849 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248559.510318 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248559.510382 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248559.511135 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248559.511187 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248575.790604 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248575.790728 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248575.790780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248606.151589 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248606.151717 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248606.151769 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248619.601700 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248619.601824 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248619.602250 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248619.602314 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248619.603067 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248619.603119 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248636.512734 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248636.512861 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248636.512915 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248666.873517 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248666.873740 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248666.873796 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248679.703659 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248679.703779 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248679.704182 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248679.704244 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248679.704995 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248679.705048 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248697.234524 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248697.234653 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248697.234708 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248727.595687 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248727.595817 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248727.595870 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248739.755755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248739.755880 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248739.756771 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248739.756824 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248739.865626 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248739.865746 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248757.956408 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248757.956542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248757.956595 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248788.317656 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248788.317784 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248788.317838 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248799.887618 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248799.887747 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248799.888510 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248799.888563 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248799.888865 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248799.889013 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248818.678404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248818.678542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248818.678607 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248849.039894 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248849.040024 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248849.040079 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248859.959613 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248859.959735 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248859.960498 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248859.960550 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248860.069650 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248860.069875 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248879.400552 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248879.400678 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248879.400732 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248909.761338 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248909.761479 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248909.761537 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248920.061583 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248920.061705 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248920.062472 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248920.062525 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248920.171575 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248920.171695 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248940.122556 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248940.122697 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248940.122751 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248970.483540 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248970.483669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248970.483723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248980.223600 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248980.223719 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248980.224483 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248980.224536 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248980.225291 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248980.225386 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249000.844582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249000.844711 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249000.844765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249031.205764 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249031.205990 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249031.206047 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249040.295520 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249040.295643 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249040.296405 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249040.296457 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249040.297130 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249040.297192 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249061.566458 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249061.566586 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249061.566640 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249091.927910 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249091.928037 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249091.928091 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249100.327467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249100.327593 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249100.328494 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249100.328547 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249100.457718 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249100.457824 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249122.288543 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249122.288671 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249122.288724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249152.649685 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249152.649816 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249152.649869 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249160.509467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249160.509544 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249160.509615 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249160.509708 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249160.510802 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249160.510855 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249183.011765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249183.011895 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249183.011948 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249213.371426 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249213.371565 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249213.371622 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249220.551434 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249220.551563 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249220.552329 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249220.552381 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249220.661500 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249220.661627 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249243.732724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249243.732852 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249243.732906 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249274.093649 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249274.093780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249274.093833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249280.663427 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249280.663550 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249280.664313 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249280.664365 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249280.773405 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249280.773526 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249304.454704 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249304.454833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249304.454886 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249334.815726 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249334.815858 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249334.815912 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249340.825357 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249340.825488 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249340.826289 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249340.826342 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249340.827122 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249340.827187 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249365.177542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249365.177669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249365.177723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249395.541404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249395.541529 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249395.541582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249482.760126 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249482.760259 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249482.760989 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249482.761042 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249484.460156 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249484.460310 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249521.879344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249521.879842 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249521.879913 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249542.731983 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249542.732513 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249542.733591 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249542.733644 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249544.402200 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249544.402588 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249552.232878 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249552.233293 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249552.233356 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249582.593375 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249582.594022 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249582.594304 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249602.934015 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249602.935138 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249602.936242 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249602.936310 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249604.514022 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249604.514411 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249612.954767 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249612.954976 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249612.955162 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249643.315410 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249643.315971 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249643.316213 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249662.938231 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249662.938636 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249662.939500 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249662.939552 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249664.606105 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249664.606520 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249673.676574 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249673.677044 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249673.677282 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249704.037945 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249704.038370 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249704.038433 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249723.107946 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249723.108405 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249723.109464 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249723.109518 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249724.667973 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249724.668503 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249734.398557 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249734.399061 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249734.399424 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249764.760077 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249764.760499 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249764.760562 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249783.149953 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249783.150509 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249783.151530 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249783.151610 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249784.860115 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249784.860543 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249795.120517 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249795.121115 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249795.121470 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249825.481800 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249825.482123 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249825.482265 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249843.301904 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249843.303003 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249843.304400 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249843.304486 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249844.971993 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249844.972487 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249855.842709 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249855.843038 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249855.843176 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249886.203512 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249886.203987 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249886.204214 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249903.433833 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249903.434091 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249903.435239 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249903.435292 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249905.023913 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249905.024305 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249916.564719 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249916.565133 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249916.565277 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249946.925416 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249946.925938 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249946.926367 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249963.525789 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249963.526251 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249963.527498 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249963.527562 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249965.085940 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249965.086355 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249977.286716 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249977.286985 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249977.287134 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250007.647650 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250007.648010 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250007.648158 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250023.557752 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250023.558227 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250023.559250 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250023.559334 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250025.247819 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250025.248795 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250038.008626 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250038.008972 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250038.009117 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250068.369715 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250068.370110 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250068.370631 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250083.689727 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250083.690303 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250083.691188 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250083.691300 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250085.359965 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250085.360376 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250098.730564 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250098.732151 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250098.732322 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250129.092925 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250129.093340 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250129.093403 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250143.761756 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250143.762207 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250143.763278 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250143.763355 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250145.471889 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250145.472434 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250159.453836 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250159.454250 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250159.454313 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250189.813551 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250189.814359 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250189.814797 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250203.923674 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250203.924112 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250203.925109 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250203.925196 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250205.513914 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250205.514315 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250220.176334 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250220.176748 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250220.176811 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250250.535671 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250250.536188 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250250.536510 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250263.945671 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250263.945967 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250263.947065 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250263.947151 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250265.655706 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250265.656171 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250280.896680 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250280.897147 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250280.897565 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250311.257655 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250311.258113 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250311.258344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250324.067765 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250324.068159 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250324.068986 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250324.069111 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250325.727674 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250325.728124 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248423.987624 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248423.987754 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248423.987809 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248439.235755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248439.235871 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248439.236632 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248439.236684 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248439.345800 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248439.345924 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248454.346566 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248454.346701 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248454.346755 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248484.708180 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248484.708309 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248484.708362 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248499.377732 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248499.377855 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248499.378618 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248499.378670 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248499.379345 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248499.379407 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248515.068667 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248515.068795 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248515.068848 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248545.429750 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248545.429879 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248545.429933 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248559.509728 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248559.509849 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248559.510318 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248559.510382 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248559.511135 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248559.511187 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248575.790604 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248575.790728 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248575.790780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248606.151589 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248606.151717 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248606.151769 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248619.601700 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248619.601824 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248619.602250 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248619.602314 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248619.603067 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248619.603119 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248636.512734 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248636.512861 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248636.512915 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248666.873517 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248666.873740 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248666.873796 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248679.703659 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248679.703779 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248679.704182 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248679.704244 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248679.704995 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248679.705048 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248697.234524 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248697.234653 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248697.234708 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248727.595687 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248727.595817 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248727.595870 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248739.755755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248739.755880 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248739.756771 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248739.756824 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248739.865626 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248739.865746 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248757.956408 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248757.956542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248757.956595 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248788.317656 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248788.317784 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248788.317838 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248799.887618 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248799.887747 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248799.888510 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248799.888563 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248799.888865 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248799.889013 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248818.678404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248818.678542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248818.678607 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248849.039894 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248849.040024 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248849.040079 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248859.959613 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248859.959735 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248859.960498 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248859.960550 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248860.069650 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248860.069875 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248879.400552 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248879.400678 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248879.400732 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248909.761338 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248909.761479 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248909.761537 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248920.061583 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248920.061705 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248920.062472 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248920.062525 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248920.171575 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248920.171695 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107248940.122556 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248940.122697 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248940.122751 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248970.483540 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107248970.483669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107248970.483723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107248980.223600 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107248980.223719 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107248980.224483 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107248980.224536 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107248980.225291 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107248980.225386 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249000.844582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249000.844711 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249000.844765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249031.205764 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249031.205990 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249031.206047 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249040.295520 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249040.295643 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249040.296405 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249040.296457 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249040.297130 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249040.297192 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249061.566458 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249061.566586 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249061.566640 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249091.927910 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249091.928037 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249091.928091 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249100.327467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249100.327593 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249100.328494 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249100.328547 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249100.457718 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249100.457824 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249122.288543 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249122.288671 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249122.288724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249152.649685 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249152.649816 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249152.649869 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249160.509467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249160.509544 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249160.509615 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249160.509708 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249160.510802 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249160.510855 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249183.011765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249183.011895 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249183.011948 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249213.371426 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249213.371565 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249213.371622 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249220.551434 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249220.551563 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249220.552329 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249220.552381 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249220.661500 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249220.661627 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249243.732724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249243.732852 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249243.732906 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249274.093649 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249274.093780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249274.093833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249280.663427 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249280.663550 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249280.664313 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249280.664365 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249280.773405 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249280.773526 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249304.454704 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249304.454833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249304.454886 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249334.815726 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249334.815858 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249334.815912 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249340.825357 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249340.825488 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249340.826289 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249340.826342 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249340.827122 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249340.827187 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249365.177542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249365.177669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249365.177723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249395.541404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249395.541529 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249395.541582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249482.760126 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249482.760259 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249482.760989 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249482.761042 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249484.460156 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249484.460310 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249521.879344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249521.879842 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249521.879913 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249542.731983 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249542.732513 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249542.733591 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249542.733644 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249544.402200 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249544.402588 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249552.232878 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249552.233293 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249552.233356 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249582.593375 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249582.594022 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249582.594304 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249602.934015 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249602.935138 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249602.936242 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249602.936310 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249604.514022 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249604.514411 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249612.954767 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249612.954976 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249612.955162 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249643.315410 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249643.315971 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249643.316213 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249662.938231 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249662.938636 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249662.939500 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249662.939552 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249664.606105 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249664.606520 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249673.676574 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249673.677044 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249673.677282 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249704.037945 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249704.038370 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249704.038433 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249723.107946 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249723.108405 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249723.109464 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249723.109518 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249724.667973 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249724.668503 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249734.398557 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249734.399061 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249734.399424 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249764.760077 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249764.760499 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249764.760562 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249783.149953 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249783.150509 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249783.151530 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249783.151610 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249784.860115 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249784.860543 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249795.120517 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249795.121115 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249795.121470 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249825.481800 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249825.482123 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249825.482265 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249843.301904 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249843.303003 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249843.304400 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249843.304486 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249844.971993 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249844.972487 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249855.842709 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249855.843038 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249855.843176 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249886.203512 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249886.203987 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249886.204214 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249903.433833 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249903.434091 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249903.435239 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249903.435292 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249905.023913 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249905.024305 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249916.564719 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249916.565133 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249916.565277 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249946.925416 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249946.925938 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249946.926367 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107249963.525789 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107249963.526251 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107249963.527498 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107249963.527562 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107249965.085940 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107249965.086355 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107249977.286716 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107249977.286985 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107249977.287134 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250007.647650 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250007.648010 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250007.648158 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250023.557752 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250023.558227 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250023.559250 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250023.559334 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250025.247819 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250025.248795 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250038.008626 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250038.008972 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250038.009117 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250068.369715 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250068.370110 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250068.370631 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250083.689727 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250083.690303 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250083.691188 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250083.691300 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250085.359965 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250085.360376 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250098.730564 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250098.732151 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250098.732322 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250129.092925 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250129.093340 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250129.093403 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250143.761756 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250143.762207 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250143.763278 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250143.763355 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250145.471889 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250145.472434 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250159.453836 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250159.454250 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250159.454313 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250189.813551 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250189.814359 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250189.814797 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250203.923674 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250203.924112 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250203.925109 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250203.925196 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250205.513914 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250205.514315 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250220.176334 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250220.176748 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250220.176811 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250250.535671 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250250.536188 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250250.536510 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250263.945671 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250263.945967 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250263.947065 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250263.947151 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250265.655706 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250265.656171 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250280.896680 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250280.897147 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250280.897565 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250311.257655 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250311.258113 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250311.258344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250324.067765 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250324.068159 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250324.068986 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250324.069111 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250325.727674 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250325.728124 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250341.618469 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250341.619001 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250341.619367 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250371.979550 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250371.980003 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250371.980254 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250384.209615 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250384.210153 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250384.211269 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250384.211322 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250385.789685 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250385.790201 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250402.340723 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250402.341145 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250402.341208 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250432.701722 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250432.702052 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250432.702193 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250444.241654 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250444.242282 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250444.243410 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250444.243524 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250445.921705 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250445.922178 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250463.062779 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250463.063193 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250463.063257 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250493.423518 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250493.423869 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250493.424011 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250504.433587 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250504.434091 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250504.435178 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250504.435231 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250506.013628 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250506.013991 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250523.791832 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250523.791966 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250523.792019 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250554.145291 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250554.145429 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250554.145494 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250564.525521 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250564.525632 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250564.526387 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250564.526439 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250566.085563 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250566.085687 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250584.506801 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250584.506957 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250584.507011 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250614.868677 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250614.868807 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250614.868861 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250624.567617 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250624.567734 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250624.568485 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250624.568536 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250626.257564 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250626.257686 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250645.229659 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250645.229812 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250645.229866 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250675.589502 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250675.589644 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250675.589708 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250684.729465 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250684.729580 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250684.730461 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250684.730512 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250686.299574 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250686.299692 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250705.950505 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250705.950692 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250705.950769 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250736.311593 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250736.311734 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250736.311798 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250744.761494 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250744.761626 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250744.762381 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250744.762433 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250746.441505 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250746.441626 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250766.672522 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250766.672708 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250766.672763 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250797.034486 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250797.034617 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250797.034670 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250804.863485 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250804.863613 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250804.864360 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250804.864411 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250806.573649 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250806.573769 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250827.395696 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250827.395823 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250827.395876 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250857.756404 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250857.756533 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250857.756585 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250865.035459 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250865.035581 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250865.036337 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250865.036388 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250866.595476 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250866.595599 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250888.116534 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250888.116674 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250888.116737 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250918.479148 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250918.479277 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250918.479332 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250925.137420 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250925.137544 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250925.138351 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250925.138403 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250926.717467 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250926.717575 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107250948.838645 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250948.838779 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250948.838836 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250979.199468 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107250979.199645 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107250979.199705 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107250985.130997 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107250985.131117 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107250985.131975 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107250985.132027 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107250986.809443 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107250986.809566 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107251009.560525 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251009.560664 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251009.560727 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251039.921739 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251039.921872 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251039.921926 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251045.311481 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107251045.311598 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107251045.312351 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107251045.312403 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107251046.871495 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107251046.871619 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107251070.282709 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251070.282849 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251070.282918 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251100.643735 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251100.643875 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251100.643928 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251105.403383 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107251105.403514 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107251105.404268 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107251105.404320 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107251106.983407 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107251106.983527 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107251131.004511 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251131.004656 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251131.004753 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251161.365529 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251161.365664 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251161.365721 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251165.455342 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107251165.455454 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107251165.456208 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107251165.456260 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107251167.135419 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107251167.135523 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107251191.726752 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251191.726890 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251191.726957 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251222.087961 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251222.088092 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251222.088146 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251225.587446 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107251225.587559 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107251225.588311 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107251225.588363 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107251227.167723 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107251227.167846 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107251252.448823 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251252.448959 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251252.449014 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251282.809626 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251282.809768 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251282.809831 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251285.629323 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107251285.629447 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107251285.630309 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107251285.630361 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107251287.319473 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107251287.319591 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107251313.170437 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251313.170575 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251313.170631 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251343.531690 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251343.531826 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251343.531880 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251345.811348 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107251345.811464 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107251345.812220 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107251345.812272 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107251347.391374 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107251347.391481 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107251373.892751 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251373.892890 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251373.892954 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251404.253471 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251404.253618 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251404.253675 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251405.923299 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107251405.923418 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107251405.924170 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107251405.924221 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107251407.513274 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107251407.513397 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107251434.614618 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251434.614759 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251434.614823 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251464.975470 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251464.975607 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251464.975662 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251466.005315 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107251466.005448 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107251466.006208 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107251466.006260 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107251467.565334 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107251467.565455 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107251495.336518 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251495.336658 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251495.336721 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251525.697793 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251525.697922 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251525.697975 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
+1107251526.047229 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0
+1107251526.047353 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0
+1107251526.048109 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0
+1107251526.048161 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0
+1107251527.737300 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0
+1107251527.737423 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0
+1107251556.058595 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0
+1107251556.058737 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0
+1107251556.058848 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0
diff --git a/logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java b/logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java
new file mode 100644
index 00000000000..d74504b5ed0
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+import com.yahoo.logserver.handlers.LogHandler;
+import com.yahoo.logserver.test.LogDispatcherTestCase;
+
+public class FlusherTestCase {
+
+ @Test
+ public void testFlusher() throws InterruptedException {
+ Flusher flusher = new Flusher();
+ LogDispatcherTestCase.MockHandler handler = new LogDispatcherTestCase.MockHandler();
+ flusher.register(handler);
+ flusher.start();
+ Thread.sleep(5000);
+ flusher.interrupt();
+ flusher.join();
+ assertTrue(handler.flushCalled>0);
+ }
+
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java
new file mode 100644
index 00000000000..1a6cd304d79
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver;
+
+import com.yahoo.log.LogSetup;
+import com.yahoo.logserver.Server;
+import com.yahoo.logserver.handlers.LogHandler;
+import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin;
+import com.yahoo.logserver.test.LogDispatcherTestCase;
+
+import java.io.IOException;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/**
+ * Unit tests for the Server class.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class ServerTestCase {
+
+ @Test
+ public void testStartupAndRegHandlers() throws IOException, InterruptedException {
+ Server.help();
+ Server server = Server.getInstance();
+ server.initialize(18322);
+ LogSetup.clearHandlers();
+ Thread serverThread = new Thread(server);
+ serverThread.start();
+ assertTrue(serverThread.isAlive());
+ LogHandler handler = new LogDispatcherTestCase.MockHandler();
+ server.registerLogHandler(handler , "foo");
+ assertEquals(Server.threadNameForHandler().get(handler), "foo");
+ server.unregisterLogHandler(handler);
+ assertEquals(Server.threadNameForHandler().get(handler), null);
+ serverThread.interrupt();
+ try {
+ serverThread.join();
+ assertTrue(true);
+ }
+ catch (InterruptedException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testPluginLoaderClassLoading() {
+ AbstractPluginLoader loader = new BuiltinPluginLoader();
+ System.setProperty("logserver.logmetrics.enable", "false");
+ loader.loadFromClass(LogMetricsPlugin.class);
+ System.setProperty("logserver.logmetrics.enable", "true");
+ loader.loadFromClass(LogMetricsPlugin.class); // Hm, no way to verify it was loaded
+ }
+
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java
new file mode 100644
index 00000000000..439c8e3978f
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.filter.test;
+
+import com.yahoo.logserver.filter.LogFilter;
+import com.yahoo.logserver.filter.LogFilterManager;
+import com.yahoo.logserver.filter.LevelFilter;
+import com.yahoo.logserver.filter.MetricsFilter;
+import com.yahoo.logserver.filter.NoMetricsFilter;
+import com.yahoo.logserver.filter.NullFilter;
+import com.yahoo.logserver.filter.MuteFilter;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+public class LogFilterManagerTestCase {
+
+ @Test
+ public void testSystemFilters () {
+ LogFilter f;
+
+ f = LogFilterManager.getLogFilter("system.allevents");
+ assertNotNull(f);
+ assertTrue(f instanceof LevelFilter);
+
+ f = LogFilterManager.getLogFilter("system.metricsevents");
+ assertNotNull(f);
+ assertTrue(f instanceof MetricsFilter);
+
+ f = LogFilterManager.getLogFilter("system.nometricsevents");
+ assertNotNull(f);
+ assertTrue(f instanceof NoMetricsFilter);
+
+
+ f = LogFilterManager.getLogFilter("system.all");
+ assertNotNull(f);
+ assertTrue(f instanceof NullFilter);
+
+ f = LogFilterManager.getLogFilter("system.mute");
+ assertNotNull(f);
+ assertTrue(f instanceof MuteFilter);
+ assertTrue(f == MuteFilter.getInstance());
+ }
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java
new file mode 100644
index 00000000000..fa22daac0d8
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.filter.test;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+
+import com.yahoo.log.event.Event;
+import com.yahoo.log.event.MalformedEventException;
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.filter.MetricsFilter;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+public class MetricsFilterTestCase {
+
+ @Test
+ public void testValueEvents ()
+ throws InvalidLogFormatException, IOException {
+ MetricsFilter filter = new MetricsFilter();
+ String filename = "src/test/files/value-events.txt";
+ BufferedReader br = new BufferedReader(new FileReader(filename));
+ for (String line = br.readLine(); line != null; line = br.readLine())
+ {
+ LogMessage m = LogMessage.parseNativeFormat(line);
+ assertNotNull(m);
+ try {
+ Event event = m.getEvent();
+ assertNotNull(event);
+ }
+ catch (MalformedEventException e) {
+ fail();
+ }
+
+ if (filter.isLoggable(m)) {
+ assertTrue(true);
+ } else {
+ fail();
+ }
+ }
+
+ }
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java
new file mode 100644
index 00000000000..5d6be5c16c7
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.filter.test;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+
+import com.yahoo.log.event.Event;
+import com.yahoo.log.event.MalformedEventException;
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.filter.MetricsFilter;
+import com.yahoo.logserver.filter.NoMetricsFilter;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/**
+ * Ensure that the NoMetricsFilter does the opposite of MetricsFilter
+ */
+public class NoMetricsFilterTestCase {
+
+ @Test
+ public void testValueEvents () throws InvalidLogFormatException, IOException {
+ NoMetricsFilter filter = new NoMetricsFilter();
+ MetricsFilter metricsFilter = new MetricsFilter();
+
+ String filename = "src/test/files/value-events.txt";
+ BufferedReader br = new BufferedReader(new FileReader(filename));
+ for (String line = br.readLine(); line != null; line = br.readLine()) {
+ LogMessage m = LogMessage.parseNativeFormat(line);
+ assertNotNull(m);
+
+ try {
+ Event event = m.getEvent();
+ assertNotNull(event);
+ }
+ catch (MalformedEventException e) {
+ fail();
+ }
+
+ if (filter.isLoggable(m)) {
+ fail();
+ } else {
+ assertTrue(true);
+ }
+
+
+ if (metricsFilter.isLoggable(m)) {
+ assertTrue(true);
+ } else {
+ fail();
+ }
+ }
+ }
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java
new file mode 100644
index 00000000000..288300f9251
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/*
+ * $Id$
+ *
+ */
+package com.yahoo.logserver.formatter.test;
+
+import com.yahoo.logserver.formatter.LogFormatter;
+import com.yahoo.logserver.formatter.LogFormatterManager;
+import com.yahoo.logserver.formatter.NullFormatter;
+import com.yahoo.logserver.formatter.TextFormatter;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/**
+ * Test the LogFormatterManager
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class LogFormatterManagerTestCase {
+
+ /**
+ * Ensure the system formatters are present
+ */
+ @Test
+ public void testSystemFormatters() {
+ LogFormatter lf = LogFormatterManager.getLogFormatter("system.textformatter");
+ assertNotNull(lf);
+ assertEquals(TextFormatter.class, lf.getClass());
+
+ lf = LogFormatterManager.getLogFormatter("system.nullformatter");
+ assertNotNull(lf);
+ assertEquals(NullFormatter.class, lf.getClass());
+ }
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java
new file mode 100644
index 00000000000..b0fc7ea09c0
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/*
+ * $Id$
+ *
+ */
+package com.yahoo.logserver.formatter.test;
+
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.formatter.NullFormatter;
+import com.yahoo.logserver.test.MockLogEntries;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/**
+ * Test the NullFormatter
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class NullFormatterTestCase {
+
+ @Test
+ public void testNullFormatter () {
+ NullFormatter nf = new NullFormatter();
+ LogMessage[] ms = MockLogEntries.getMessages();
+ for (LogMessage m : ms) {
+ assertEquals(m.toString(), nf.format(m));
+ }
+ }
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java
new file mode 100644
index 00000000000..2e7e2d38b57
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/*
+ * $Id$
+ *
+ */
+package com.yahoo.logserver.formatter.test;
+
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.formatter.TextFormatter;
+import com.yahoo.logserver.test.MockLogEntries;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/**
+ * Test the TextFormatter
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class TextFormatterTestCase {
+
+ /**
+ * Just simple test to make sure it doesn't die on us
+ */
+ @Test
+ public void testTextFormatter () {
+ TextFormatter tf = new TextFormatter();
+ LogMessage[] ms = MockLogEntries.getMessages();
+ for (int i = 0; i < ms.length; i++) {
+ System.out.println(tf.format(ms[i]));
+ }
+ }
+
+ /**
+ * Test that a specific log message is formatted correctly
+ */
+ @Test
+ public void testSpecificMessage () throws InvalidLogFormatException {
+ String l = "1115200798.195568\texample.yahoo.com\t65819\ttopleveldispatch\tfdispatch.queryperf\tevent\tvalue/1 name=\"query_eval_time_avg_s\" value=0.0229635972697721825";
+ String result = "2005-05-04 09:59:58 example.yahoo.com 65819 topleveldispatch fdispatch.queryperf EVENT value/1 name=\"query_eval_time_avg_s\" value=0.0229635972697721825\n";
+ LogMessage m = LogMessage.parseNativeFormat(l);
+ TextFormatter tf = new TextFormatter();
+ assertEquals(result, tf.format(m));
+ }
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java
new file mode 100644
index 00000000000..16c1c780bb1
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java
@@ -0,0 +1,96 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+
+import org.junit.Test;
+
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.test.LogDispatcherTestCase;
+
+public class HandlerThreadTestCase {
+
+ @Test
+ public void testHandlerThread() throws InterruptedException, InvalidLogFormatException {
+ HandlerThread thread = new HandlerThread("myThread");
+ LogDispatcherTestCase.MockHandler h1 = new LogDispatcherTestCase.MockHandler();
+ LogDispatcherTestCase.MockHandler h2 = new LogDispatcherTestCase.MockHandler();
+ LogDispatcherTestCase.MockHandler h3 = new LogDispatcherTestCase.MockHandler();
+ thread.registerHandler(h1);
+ thread.registerHandler(h2);
+ thread.registerHandler(h3);
+ thread.start();
+ String msgstr1 = "1098709001\t"
+ + "nalle.puh.com\t"
+ + "23234\t"
+ + "serviceName\t"
+ + "tst\t"
+ + "info\t"
+ + "this is a test";
+ String msgstr2 = "1098709002\t"
+ + "nalle.puh.com\t"
+ + "23234\t"
+ + "serviceName1\t"
+ + "tst\t"
+ + "info\t"
+ + "this is a test too";
+ String msgstr3 = "1098709003\t"
+ + "nalle.puh.com\t"
+ + "23234\t"
+ + "serviceName3\t"
+ + "tst\t"
+ + "info\t"
+ + "this is a test also";
+ final LogMessage msg1 = LogMessage.parseNativeFormat(msgstr1);
+ final LogMessage msg2 = LogMessage.parseNativeFormat(msgstr2);
+ final LogMessage msg3 = LogMessage.parseNativeFormat(msgstr3);
+ thread.handle(msg1);
+ while ((h1.messages.size() < 1) || (h2.messages.size() < 1) || (h3.messages.size() < 1)) {
+ thread.sleep(10);
+ }
+ assertEquals(h1.messages.size(), 1);
+ assertEquals(h2.messages.size(), 1);
+ assertEquals(h3.messages.size(), 1);
+ thread.handle(new ArrayList<LogMessage>() {{add(msg1); add(msg2); add(msg3); }});
+ while ((h1.messages.size() < 4) || (h2.messages.size() < 4) || (h3.messages.size() < 4)) {
+ thread.sleep(10);
+ }
+ assertEquals(h1.messages.size(), 4);
+ assertEquals(h2.messages.size(), 4);
+ assertEquals(h3.messages.size(), 4);
+ assertTrue(thread.getQueue().isEmpty());
+ thread.unregisterHandler(h3);
+ assertEquals(thread.getHandlers().length, 2);
+ thread.flush();
+ thread.close();
+ thread.interrupt();
+ thread.join();
+ }
+
+ @Test
+ public void testAbortThread() throws InvalidLogFormatException, InterruptedException {
+ HandlerThread thread = new HandlerThread("myThread");
+ LogDispatcherTestCase.MockHandler h1 = new LogDispatcherTestCase.MockHandler();
+ thread.registerHandler(h1);
+ thread.start();
+ String msgstr1 = "1098709001\t"
+ + "nalle.puh.com\t"
+ + "23234\t"
+ + "serviceName\t"
+ + "tst\t"
+ + "info\t"
+ + "this is a test";
+ final LogMessage msg1 = LogMessage.parseNativeFormat(msgstr1);
+ thread.handle(msg1);
+ while (h1.messages.size() < 1) {
+ thread.sleep(10);
+ }
+ assertEquals(h1.messages.size(), 1);
+ thread.interrupt();
+ thread.join();
+ }
+
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java
new file mode 100644
index 00000000000..967441ce250
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java
@@ -0,0 +1,115 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.lasterrorsholder;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.Server;
+import com.yahoo.text.Utf8;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.SocketChannel;
+import java.time.Duration;
+import java.time.Instant;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+public class LastErrorsHolderTestCase {
+ private static final int serverPort = 18324;
+ private static final int lastErrorsHolderPort = 18326;
+ Server server;
+ Thread serverThread;
+ LastErrorsHolder lastErrorsHolder;
+
+ @Before
+ public void setUp() throws InterruptedException, IOException {
+ server = Server.getInstance();
+ server.initialize(serverPort);
+ serverThread = new Thread(server);
+ serverThread.start();
+ lastErrorsHolder = new LastErrorsHolder(lastErrorsHolderPort);
+ }
+
+ @After
+ public void tearDown() throws InterruptedException {
+ if (serverThread!=null) {
+ serverThread.interrupt();
+ serverThread.join();
+ }
+ if (lastErrorsHolder !=null) lastErrorsHolder.close();
+ }
+
+ public String connectAndGetLogMessages() throws InterruptedException, IOException {
+ SocketChannel socket = null;
+ Instant start = Instant.now();
+ while (Instant.now().isBefore(start.plus(Duration.ofMinutes(1)))) {
+ try {
+ socket = SocketChannel.open(new InetSocketAddress(lastErrorsHolderPort));
+ break;
+ } catch (Exception e) {
+ Thread.sleep(100);
+ }
+ }
+ if (socket == null) {
+ throw new RuntimeException("Could not connect to server");
+ }
+
+ ByteBuffer buf = ByteBuffer.allocateDirect(10000);
+ int bytesRead = socket.read(buf);
+ byte[] bytes = new byte[bytesRead];
+ buf.position(0);
+ buf.get(bytes);
+ socket.close();
+
+ return Utf8.toString(bytes);
+ }
+
+
+ @Test
+ public void testLastErrorsHolder() throws IOException, InvalidLogFormatException, InterruptedException {
+ LastErrorsHolder.Message logMessage1 = new LastErrorsHolder.Message(1433996283, "host1.yahoo.com", "container", LogLevel.ERROR.getName(), "foo");
+ LastErrorsHolder.Message logMessage2 = new LastErrorsHolder.Message(1433996284, "host2.yahoo.com", "container", LogLevel.ERROR.getName(), "bar");
+ LastErrorsHolder.Message logMessage3 = new LastErrorsHolder.Message(1433996285, "host2.yahoo.com", "container", LogLevel.INFO.getName(), "bar");
+
+ LastErrorsHolder.Messages messages = new LastErrorsHolder.Messages();
+
+ // No log messages yet
+ String logs = connectAndGetLogMessages();
+ final ObjectMapper mapper = new ObjectMapper();
+ StringWriter stringWriter = new StringWriter();
+ mapper.writeValue(stringWriter, messages);
+ assertThat(logs, is(stringWriter.toString()));
+
+ // Three messages, one is at level INFO
+ lastErrorsHolder.doHandle(createLogMessage(logMessage1));
+ lastErrorsHolder.doHandle(createLogMessage(logMessage2));
+ lastErrorsHolder.doHandle(createLogMessage(logMessage3));
+ messages = new LastErrorsHolder.Messages();
+ messages.addMessage(logMessage1);
+ messages.addMessage(logMessage2);
+ messages.setNumberOfErrors(2);
+ // Not adding logMessage3, since it is at level INFO
+
+ logs = connectAndGetLogMessages();
+ stringWriter = new StringWriter();
+ mapper.writeValue(stringWriter, messages);
+ assertThat(logs, is(stringWriter.toString()));
+ }
+
+ private LogMessage createLogMessage(LastErrorsHolder.Message message) throws InvalidLogFormatException {
+ return createLogMessage(message.getTime(), message.getHostname(), message.getService(), message.getLogLevel(), message.getMessage());
+ }
+
+ private LogMessage createLogMessage(long time, String hostname, String service, String logLevel, String message) throws InvalidLogFormatException {
+ return LogMessage.parseNativeFormat(String.format("%d\t%s\t1/1\t%s\tcomponent\t%s\t%s", time, hostname, service, logLevel.toLowerCase(), message));
+ }
+
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java
new file mode 100644
index 00000000000..686c8a8a089
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java
@@ -0,0 +1,122 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.logmetrics.test;
+
+import java.util.Map;
+
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.handlers.logmetrics.LogMetricsHandler;
+import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin;
+import com.yahoo.plugin.SystemPropertyConfig;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:musum@europe.yahoo-inc.com">Harald Musum</a>
+ */
+public class LogMetricsTestCase {
+ // Some of the tests depend upon the number of messages for a
+ // host, log level etc. to succeed, so you may have update the
+ // tests if you change something in mStrings. config, debug and
+ // spam are filtered out and not handled.
+ private static final String[] mStrings = {
+ "1095159244.095\thostA\t1/2\tservice\tcomponent\tconfig\tpayload1",
+ "1095206399.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload2",
+ "1095206400.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload3",
+ "1095206401.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload4",
+ "1095206402.000\thostA\t1/2\tservice\tcomponent\twarning\tpayload5",
+ "1095206403.000\thostA\t1/2\tservice\tcomponent\terror\tpayload6",
+ "1095206404.000\thostB\t1/2\tservice\tcomponent\tinfo\tpayload7",
+ "1095206405.000\thostB\t1/2\tservice\tcomponent\tfatal\tpayload8",
+ "1095206406.000\thostB\t1/2\tservice\tcomponent\tdebug\tpayload9",
+ };
+
+ private static final LogMessage[] msg = new LogMessage[mStrings.length];
+
+ static {
+ try {
+ for (int i = 0; i < mStrings.length; i++) {
+ msg[i] = LogMessage.parseNativeFormat(mStrings[i]);
+ }
+ } catch (InvalidLogFormatException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Log some messages to the handler and verify that they are
+ * counted by the handler and that the count for each level is
+ * correct.
+ */
+ @Test
+ public void testLevelCountTotal () throws java.io.IOException, InvalidLogFormatException {
+ LogMetricsHandler a = new LogMetricsHandler();
+
+ for (LogMessage aMsg : msg) {
+ a.handle(aMsg);
+ }
+
+ long count = a.getMetricsCount();
+ a.close();
+ // Not all messages are processes (debug and spam)
+ assertEquals(count, 7);
+ }
+
+
+
+ /**
+ * Log some messages to the handler and verify that they are
+ * counted by the handler and that the count for each level is
+ * correct (the count for each host is summed into one count for
+ * each level).
+ */
+ @Test
+ public void testLevelCountAggregated () throws java.io.IOException, InvalidLogFormatException {
+ LogMetricsHandler a = new LogMetricsHandler();
+
+ for (LogMessage aMsg : msg) {
+ a.handle(aMsg);
+ }
+
+ Map<String, Long> levelCount = a.getMetricsPerLevel();
+ assertEquals(levelCount.entrySet().size(), 5);
+ for (Map.Entry<String, Long> entry : levelCount.entrySet()) {
+ String key = entry.getKey();
+ if (key.equals("config")) {
+ assertEquals(entry.getValue(), new Long(1));
+ } else if (key.equals("info")) {
+ assertEquals(entry.getValue(), new Long(4));
+ } else if (key.equals("warning")) {
+ assertEquals(entry.getValue(), new Long(1));
+ } else if (key.equals("severe")) {
+ assertEquals(entry.getValue(), new Long(0));
+ } else if (key.equals("error")) {
+ assertEquals(entry.getValue(), new Long(1));
+ } else if (key.equals("fatal")) {
+ assertEquals(entry.getValue(), new Long(1));
+ } else if (key.equals("debug")) {
+ assertEquals(entry.getValue(), new Long(0)); // always 0
+ }
+ }
+ a.close();
+ }
+
+ @Test
+ public void testLogMetricsPlugin() {
+ LogMetricsPlugin lp = new LogMetricsPlugin();
+ try {
+ lp.shutdownPlugin();
+ fail("Shutdown before init didn't throw.");
+ } catch (Exception e) {
+ }
+ lp.initPlugin(new SystemPropertyConfig("test"));
+ try {
+ lp.initPlugin(new SystemPropertyConfig("test"));
+ fail("Multiple init didn't throw.");
+ } catch (Exception e) {
+ }
+ lp.shutdownPlugin();
+ }
+
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java
new file mode 100644
index 00000000000..b0d846a2965
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/* -*- c-basic-offset: 4 -*-
+ *
+ * $Id$
+ *
+ */
+package com.yahoo.logserver.handlers.replicator;
+
+import java.nio.ByteBuffer;
+import com.yahoo.logserver.handlers.replicator.FormattedBufferCache;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.formatter.LogFormatter;
+import com.yahoo.logserver.formatter.LogFormatterManager;
+import com.yahoo.logserver.test.MockLogEntries;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/**
+ * Unit tests for thge LogMessage class.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class FormattedBufferCacheTestCase {
+
+ @Test
+ public void testCache () {
+ LogMessage msgs[] = MockLogEntries.getMessages();
+ FormattedBufferCache cache = new FormattedBufferCache();
+ String n[] = LogFormatterManager.getFormatterNames();
+ for (int i = 0; i < n.length; i++) {
+ LogFormatter f = LogFormatterManager.getLogFormatter(n[i]);
+ for (int j = 0; j < msgs.length; j++) {
+ ByteBuffer bb = cache.getFormatted(msgs[j], f);
+ assertNotNull(bb);
+ }
+ }
+
+ assertTrue(cache.getUnderlyingMapOnlyForTesting().size() > 0);
+ cache.reset();
+ assertTrue(cache.getUnderlyingMapOnlyForTesting().size() == 0);
+ }
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java
new file mode 100644
index 00000000000..dbb0b7817f6
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.replicator;
+
+import static org.junit.Assert.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.yahoo.plugin.SystemPropertyConfig;
+
+public class ReplicatorPluginTestCase {
+
+ ReplicatorPlugin rp;
+
+ @Before
+ public void setUp() {
+ rp = new ReplicatorPlugin();
+ }
+
+ @After
+ public void tearDown() {
+ if (rp!=null) {
+ rp.shutdownPlugin();
+ }
+ }
+
+ @Test
+ public void testReplicatorPlugin() {
+ System.setProperty("replicatorplugin.test.port", "18325");
+ try {
+ rp.shutdownPlugin();
+ fail("Shutdown before init didn't throw.");
+ } catch (Exception e) {
+ }
+ rp.initPlugin(new SystemPropertyConfig("replicatorplugin.test."));
+ try {
+ rp.initPlugin(new SystemPropertyConfig("test"));
+ fail("Multiple init didn't throw.");
+ } catch (Exception e) {
+ }
+ }
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java
new file mode 100644
index 00000000000..37f25e47a1c
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java
@@ -0,0 +1,134 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.handlers.replicator;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.SocketChannel;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.yahoo.io.Listener;
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.Server;
+import com.yahoo.logserver.filter.LogFilter;
+import com.yahoo.logserver.filter.LogFilterManager;
+import com.yahoo.logserver.formatter.LogFormatterManager;
+
+public class ReplicatorTestCase {
+ Server server;
+ Thread serverThread;
+ Replicator replicator;
+ private SocketChannel socket;
+ private Listener listener;
+ private ReplicatorConnection conn;
+
+ @Before
+ public void setUp() throws InterruptedException, IOException {
+ server = Server.getInstance();
+ server.initialize(18321);
+ serverThread = new Thread(server);
+ serverThread.start();
+ long start = System.currentTimeMillis();
+ long timeout = 60000;
+ while (System.currentTimeMillis() < (start + timeout)) {
+ try {
+ socket = SocketChannel.open(new InetSocketAddress(18321));
+ break;
+ } catch (Exception e) {
+ Thread.sleep(100);
+ }
+ }
+ listener = new Listener("test");
+ replicator = new Replicator(18323);
+ conn = (ReplicatorConnection) replicator.newConnection(socket, listener);
+ }
+
+ @After
+ public void tearDown() throws InterruptedException {
+ if (serverThread!=null) {
+ serverThread.interrupt();
+ serverThread.join();
+ }
+ if (replicator!=null) replicator.close();
+ }
+
+ @Test
+ public void testReplicator() throws IOException, InvalidLogFormatException {
+ LogMessage msg = LogMessage.
+ parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\tevent\tfoo");
+ assertFalse(conn.isLoggable(msg)); // Default all muted
+ conn.onUse("system.all");
+ assertTrue(conn.isLoggable(msg));
+ assertTrue(conn.getTotalBytesWritten()>50); // Should be in this ballpark
+ conn.onCommand("use system.mute");
+ assertFalse(conn.isLoggable(msg));
+ assertEquals("system.mute", conn.getLogFilterName());
+ replicator.doHandle(msg);
+ conn.read();
+ assertEquals(2, conn.getNumHandled());
+ conn.onFormatter("system.textformatter");
+ assertEquals(conn.formatter, LogFormatterManager.getLogFormatter("system.textformatter"));
+ conn.onCommand("formatter system.nullformatter");
+ assertEquals(conn.formatter, LogFormatterManager.getLogFormatter("system.nullformatter"));
+ assertEquals(4, conn.getNumHandled());
+ conn.onList();
+ assertEquals(11, conn.getNumHandled()); // 6 filters + start/stop msg
+ conn.onCommand("list");
+ assertEquals(18, conn.getNumHandled()); // 6 filters + start/stop msg
+ conn.onListFormatters();
+ assertEquals(22, conn.getNumHandled()); // 4 formatters
+ conn.onCommand("listformatters");
+ assertEquals(26, conn.getNumHandled()); // 4 formatters
+
+ conn.onStats();
+ assertEquals(27, conn.getNumHandled()); // 1 line
+ conn.onCommand("stats");
+ assertEquals(28, conn.getNumHandled()); // 1 line
+ conn.enqueue(ByteBuffer.wrap("baz".getBytes()));
+ assertEquals(29, conn.getNumHandled()); // 1 line
+ conn.onCommand("ping");
+ assertEquals(30, conn.getNumHandled()); // 1 line
+ conn.onCommand("quit");
+ assertEquals(31, conn.getNumHandled()); // 1 line
+ assertEquals(0, conn.getNumDropped());
+ assertEquals(0, conn.getNumQueued());
+ LogFilterManager.addLogFilter("test.onlyerror", new LogFilter() {
+ @Override
+ public boolean isLoggable(LogMessage msg) {
+ return msg.getLevel().equals(LogLevel.ERROR);
+ }
+
+ @Override
+ public String description() {
+ return "Only error";
+ }
+ });
+ conn.onUse("test.onlyerror");
+ assertFalse(conn.isLoggable(LogMessage.
+ parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\tdebug\tfoo")));
+ assertTrue(conn.isLoggable(LogMessage.
+ parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar")));
+ assertEquals(conn.selectOps(), 1);
+ assertEquals(conn.description(), "Only error");
+ conn.setFilter(null);
+ assertTrue(conn.isLoggable(LogMessage.
+ parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar")));
+ assertEquals(conn.description(), "No filter defined");
+ assertEquals(conn.getRemoteHost(), "0.0.0.0");
+ conn.onFormatter("nonexistant");
+ assertEquals(conn.formatter, LogFormatterManager.getLogFormatter("system.nullformatter")); // unchanged
+ conn.onUse("nonexistant");
+ assertTrue(conn.isLoggable(LogMessage.
+ parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar")));
+ conn.close();
+ }
+
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/status/.gitignore b/logserver/src/test/java/com/yahoo/logserver/handlers/status/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/status/.gitignore
diff --git a/logserver/src/test/java/com/yahoo/logserver/httphandlers/.gitignore b/logserver/src/test/java/com/yahoo/logserver/httphandlers/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/httphandlers/.gitignore
diff --git a/logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java b/logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java
new file mode 100644
index 00000000000..5f0897187c4
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java
@@ -0,0 +1,92 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.net.control.test;
+
+import com.yahoo.logserver.net.control.Levels;
+import com.yahoo.logserver.net.control.State;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class LevelsTestCase {
+ /**
+ * Make sure the parsing works
+ */
+ @Test
+ public void testParser() {
+ Levels levels = Levels.parse("fatal=forward,error=forward,warning=noforward");
+ assertNotNull(levels);
+ assertSame(State.FORWARD, levels.getLevelState("fatal"));
+ assertSame(State.FORWARD, levels.getLevelState("error"));
+ assertSame(State.NOFORWARD, levels.getLevelState("warning"));
+ }
+
+ /**
+ * Ensure that the Levels are in a known default state. A Levels
+ * object that was newly created is in a defined default state
+ * which has been deemed to be "reasonable". In general, it will
+ * specify forwarding all log levels except "debug" and it will
+ * specify "spam" to be turned off.
+ *
+ */
+ @Test
+ public void testDefaultLevels () {
+ Levels levels = new Levels();
+ assertSame(State.FORWARD, levels.getLevelState("event"));
+ assertSame(State.FORWARD, levels.getLevelState("fatal"));
+ assertSame(State.FORWARD, levels.getLevelState("error"));
+ assertSame(State.FORWARD, levels.getLevelState("warning"));
+ assertSame(State.FORWARD, levels.getLevelState("info"));
+ assertSame(State.FORWARD, levels.getLevelState("config"));
+ assertSame(State.NOFORWARD, levels.getLevelState("debug"));
+ assertSame(State.NOFORWARD, levels.getLevelState("spam"));
+ }
+
+ /**
+ * This test also documents/verifies the default behavior
+ * of the Levels class.
+ *
+ */
+ @Test
+ public void testToString() {
+ String in = "fatal=forward,error=forward,warning=noforward";
+ String out = "event=forward,fatal=forward,error=forward,warning=noforward,info=forward,config=forward,debug=noforward,spam=noforward";
+ Levels levels = Levels.parse(in);
+ assertEquals(out, levels.toString());
+
+ }
+
+ /**
+ * Clone testing
+ */
+ @Test
+ public void testClone () {
+ Levels l1 = Levels.parse("error=noforward");
+ assertEquals(l1.toString(), l1.clone().toString());
+ assertSame(State.NOFORWARD, l1.getLevelState("error"));
+ assertSame(State.NOFORWARD, ((Levels)l1.clone()).getLevelState("error"));
+ assertSame(l1.getLevelState("error"),
+ ((Levels)l1.clone()).getLevelState("error"));
+ }
+
+ /**
+ * test parser
+ */
+ @Test
+ public void testUpdateLevels () {
+ Levels l1 = Levels.parse("error=noforward");
+ assertSame(State.NOFORWARD, l1.getLevelState("error"));
+
+ // should be unaffected
+ assertSame(State.FORWARD, l1.getLevelState("warning"));
+
+ // update and check that the update worked
+ l1.updateLevels("error=noforward");
+ assertSame(State.NOFORWARD, l1.getLevelState("error"));
+
+ // should be unaffected
+ assertSame(State.FORWARD, l1.getLevelState("warning"));
+ }
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java b/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java
new file mode 100644
index 00000000000..06f35567e9f
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java
@@ -0,0 +1,211 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.net.test;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.LogDispatcher;
+import com.yahoo.logserver.handlers.AbstractLogHandler;
+import com.yahoo.logserver.net.LogConnection;
+import com.yahoo.logserver.test.MockLogEntries;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.charset.Charset;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.logging.Logger;
+
+import static org.junit.Assert.*;
+
+/**
+ * Unit tests for the log connection class. Wow is this too
+ * complex!
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class LogConnectionTestCase {
+ private static final Logger log = Logger.getLogger(LogConnectionTestCase.class.getName());
+
+ private static final int PROBING_RANGE_START = 41352;
+ private static final Charset charset = Charset.forName("utf-8");
+ private static final ByteBuffer bigBuffer;
+ private int port;
+
+ static {
+ StringBuilder sb = new StringBuilder(LogConnection.READBUFFER_SIZE * 6)
+ .append(MockLogEntries.getMessages()[0].toString())
+ .append(MockLogEntries.getMessages()[1].toString())
+ .append(MockLogEntries.getMessages()[2].toString());
+
+ // get a valid log message prefix
+ String prefix = MockLogEntries.getMessages()[2].toString();
+ prefix = prefix.substring(0, prefix.length() - 1);
+ sb.append(prefix);
+
+ // fill up the remainding buffer with rubbish to make
+ // it too long
+ for (int i = 0; i < (LogConnection.READBUFFER_SIZE * 3); i++) {
+ sb.append("a");
+ }
+ sb.append('\n');
+ sb.append(MockLogEntries.getMessages()[2].toString());
+ bigBuffer = charset.encode(sb.toString());
+ }
+
+ /**
+ * this code is nothing short of completely hideous. the exception
+ * handling is awful and the code is messy, but it should be a fairly
+ * efficient and robust way of testing buffer overflow conditions.
+ */
+ @Test
+ public void testOverflow() {
+ final CyclicBarrier barrier = new CyclicBarrier(2);
+
+
+ // this inner class is used to define a mock handler
+ // which will help us examine the messages actually
+ // handled.
+ class MockHandler extends AbstractLogHandler {
+ private final List<LogMessage> messages = new LinkedList<LogMessage>();
+
+ public boolean doHandle(LogMessage msg) {
+ messages.add(msg);
+ return true;
+ }
+
+ public List<LogMessage> getMessages() {
+ return messages;
+ }
+
+ public void flush() {
+ }
+
+ public void close() {
+ }
+
+ public String toString() {
+ return null;
+ }
+ }
+
+ Thread serverThread = new Thread() {
+ public void run() {
+ ServerSocketChannel ss = setUpListenSocket();
+ if (ss == null) {
+ fail("unable to set up listen socket");
+ return;
+ }
+
+ setPort(ss.socket().getLocalPort());
+
+ // listen port is up now so we can trigger the barrier
+ try {
+ barrier.await();
+
+ while (!Thread.currentThread().isInterrupted()) {
+ SocketChannel s = ss.accept();
+ pushBigBuffer(s);
+ s.close();
+ }
+ } catch (BrokenBarrierException e) {
+ fail(e.getMessage());
+ return;
+ } catch (InterruptedException | java.nio.channels.ClosedByInterruptException e) {
+ return;
+ } catch (IOException e) {
+ log.log(LogLevel.ERROR, "argh", e);
+ fail();
+ return;
+ }
+
+ }
+ };
+ serverThread.start();
+ assertTrue(serverThread.isAlive());
+
+ try {
+ barrier.await();
+ } catch (BrokenBarrierException e) {
+ fail(e.getMessage());
+ return;
+ } catch (InterruptedException e) {
+ return;
+ }
+
+ SocketChannel sock;
+ try {
+ sock = SocketChannel.open(new InetSocketAddress("localhost", port));
+ } catch (IOException e) {
+ fail(e.getMessage());
+ return;
+ }
+
+ LogDispatcher dispatcher = new LogDispatcher();
+ MockHandler mock = new MockHandler();
+ assertTrue(mock.getName().endsWith("MockHandler"));
+ dispatcher.registerLogHandler(mock);
+ LogConnection logConnection =
+ new LogConnection(sock, null, dispatcher, null);
+
+ try {
+ for (int i = 0; i < 100; i++) {
+ logConnection.read();
+ }
+ } catch (java.nio.channels.ClosedChannelException e) {
+ // ignore, this is normal
+ } catch (IOException e) {
+ log.log(LogLevel.ERROR, "error during reading", e);
+ }
+
+ // there should be 5 messages
+ assertEquals(5, mock.getMessages().size());
+ assertEquals(5, mock.getCount());
+
+ // the 4'th message should be long
+ String m = (mock.getMessages().get(3)).getPayload();
+ assertTrue(m.length() > 10000);
+
+ serverThread.interrupt();
+ try {
+ serverThread.join();
+ assertTrue(true);
+ } catch (InterruptedException e) {
+ fail();
+ }
+ }
+
+ private void pushBigBuffer(SocketChannel socket) {
+ try {
+ ByteBuffer slice = bigBuffer.slice();
+ while (slice.hasRemaining()) {
+ @SuppressWarnings("unused")
+ int ret = socket.write(slice);
+ }
+ } catch (java.io.IOException e) {
+
+ }
+ }
+
+ private void setPort(int port) {
+ this.port = port;
+ }
+
+ private ServerSocketChannel setUpListenSocket() {
+ int p = 18327;
+ try {
+ ServerSocketChannel s = ServerSocketChannel.open();
+ s.socket().setReuseAddress(true);
+ s.socket().bind(new InetSocketAddress("127.0.0.1", p));
+ return s;
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ return null;
+ }
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java b/logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java
new file mode 100644
index 00000000000..37ad625247d
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java
@@ -0,0 +1,131 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.logserver.LogDispatcher;
+import com.yahoo.log.LogMessage;
+import com.yahoo.logserver.handlers.LogHandler;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/**
+ * Unit tests for thge LogMessage class.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class LogDispatcherTestCase {
+ private static LogMessage sample1;
+ private static LogMessage sample2;
+ static {
+ try {
+ sample1 = LogMessage.parseNativeFormat("1096639280.524133 malfunction 26851 - logtest info Starting up, called as ./log/logtest");
+ sample2 = LogMessage.parseNativeFormat("1096639280.524133 malfunction 26851 - logtest info More crap");
+ }
+ catch (InvalidLogFormatException e) {
+ assertTrue(false);
+ }
+ }
+
+ public static class MockHandler implements LogHandler {
+ public final List<LogMessage> messages= new ArrayList<LogMessage>(5);
+ public int flushCalled = 0;
+ public int closeCalled = 0;
+
+ public void handle (LogMessage msg) {
+ messages.add(msg);
+ }
+
+ public void handle (List<LogMessage> messages) {
+ for (LogMessage lm : messages) {
+ handle(lm);
+ }
+ }
+
+ public void flush () {
+ flushCalled++;
+ }
+ public void close () {
+ closeCalled++;
+ }
+ public String getName () {
+ return MockHandler.class.getName();
+ }
+ }
+
+ @Test
+ public void testLogDispatcherBatchMode() {
+ MockHandler handler = new MockHandler();
+ LogDispatcher dispatcher = new LogDispatcher();
+ dispatcher.setBatchedMode(true);
+ dispatcher.registerLogHandler(handler);
+ assertEquals(0, dispatcher.getMessageCount());
+ dispatcher.handle(sample1);
+ assertEquals(1, dispatcher.getMessageCount());
+ dispatcher.handle(sample2);
+ assertEquals(2, dispatcher.getMessageCount());
+ assertEquals(0, handler.messages.size());
+ dispatcher.flush();
+ assertEquals(2, handler.messages.size());
+ }
+
+ @Test
+ public void testTestLogHandlerRegistration () {
+ MockHandler handler = new MockHandler();
+ LogDispatcher dispatcher = new LogDispatcher();
+ dispatcher.registerLogHandler(handler);
+
+ assertNotNull(dispatcher.getLogHandlers());
+ LogHandler[] handlers = dispatcher.getLogHandlers();
+ assertNotNull(handlers);
+ assertEquals(1, handlers.length);
+ assertTrue(handler == handlers[0]);
+ }
+
+ @Test
+ public void testMessageCount () {
+ MockHandler handler = new MockHandler();
+ LogDispatcher dispatcher = new LogDispatcher();
+ dispatcher.registerLogHandler(handler);
+ assertEquals(0, dispatcher.getMessageCount());
+
+ dispatcher.handle(sample1);
+ assertEquals(1, dispatcher.getMessageCount());
+
+ dispatcher.handle(sample2);
+ assertEquals(2, dispatcher.getMessageCount());
+
+ assertEquals(2, handler.messages.size());
+ }
+
+ @Test
+ public void testVerifyMessages () {
+ MockHandler handler = new MockHandler();
+ LogDispatcher dispatcher = new LogDispatcher();
+ dispatcher.registerLogHandler(handler);
+
+ dispatcher.handle(sample1);
+ dispatcher.handle(sample2);
+
+ assertTrue(sample1 == handler.messages.get(0));
+ assertTrue(sample2 == handler.messages.get(1));
+ }
+
+ // TODO: this test makes very little sense until we refactor a bit
+ @Test
+ public void testClose () {
+ MockHandler handler = new MockHandler();
+ LogDispatcher dispatcher = new LogDispatcher();
+ dispatcher.registerLogHandler(handler);
+
+ assertEquals(0, handler.flushCalled);
+ assertEquals(0, handler.closeCalled);
+
+ dispatcher.close();
+ assertEquals(0, handler.closeCalled);
+ }
+
+}
diff --git a/logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java b/logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java
new file mode 100644
index 00000000000..a9bb8084a7c
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.test;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.zip.GZIPInputStream;
+
+import com.yahoo.log.event.Event;
+import com.yahoo.log.event.MalformedEventException;
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogMessage;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/**
+ * Unit tests for thge LogMessage class.
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class LogMessageTestCase {
+
+ /**
+ * The original test was rubbish. We just test that parsing
+ * is okay here. The way we do it is to check that we have
+ * some log messages available from the MockLogEntries class.
+ * If there are none the parsing failed.
+ */
+ @Test
+ public void testLogParsing ()
+ {
+ assertTrue(MockLogEntries.getMessages().length > 0);
+ }
+
+ /**
+ * Read in some events and make sure we are able to identify
+ * them as such.
+ */
+ @Test
+ public void testEvents () throws IOException {
+ String eventfile = "src/test/files/event.txt.gz";
+ BufferedReader br = new BufferedReader(
+ new InputStreamReader(
+ new GZIPInputStream(
+ new FileInputStream(eventfile))));
+
+ for (String line = br.readLine(); line != null; line = br.readLine()) {
+ try {
+ LogMessage m = LogMessage.parseNativeFormat(line);
+ try {
+ Event event = m.getEvent();
+ assertNotNull(event);
+ } catch (MalformedEventException evx) {
+ fail();
+ }
+ } catch (InvalidLogFormatException e) {
+ fail();
+ }
+ }
+ }
+}
+
diff --git a/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java b/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java
new file mode 100644
index 00000000000..18cf8afb8a0
--- /dev/null
+++ b/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.logserver.test;
+
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogMessage;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * This class is used to gain access to a bunch of log entries
+ * so we can use the same log messages in several different tests
+ *
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ */
+public class MockLogEntries {
+ private static final MockLogEntries instance = new MockLogEntries();
+
+ private final LogMessage[] messages;
+
+ /**
+ * Private constructor which reads the log messages and builds
+ * an array of LogMessage entries.
+ */
+ private MockLogEntries () {
+ List<LogMessage> msgs = new LinkedList<LogMessage>();
+ try {
+ String name = "src/test/files/logEntries.txt";
+ BufferedReader br = new BufferedReader(new FileReader(name));
+ for (String line = br.readLine(); line != null; line = br.readLine()) {
+ LogMessage m = LogMessage.parseNativeFormat(line);
+ msgs.add(m);
+ }
+ } catch (InvalidLogFormatException | IOException e) {
+ // do nothing
+ }
+
+ LogMessage[] m = new LogMessage[msgs.size()];
+ msgs.toArray(m);
+ messages = m;
+ }
+
+ /**
+ * Return the LogMessage instances we've constructed from the
+ * stored log messages we have.
+ */
+ public static LogMessage[] getMessages() {
+ return instance.messages;
+ }
+}