From 72231250ed81e10d66bfe70701e64fa5fe50f712 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 15 Jun 2016 23:09:44 +0200 Subject: Publish --- logserver/.gitignore | 16 + logserver/OWNERS | 1 + logserver/bin/logserver-start.sh | 75 ++ logserver/pom.xml | 87 ++ .../com/yahoo/logserver/AbstractPluginLoader.java | 46 + .../com/yahoo/logserver/BuiltinPluginLoader.java | 31 + .../src/main/java/com/yahoo/logserver/Flusher.java | 57 ++ .../java/com/yahoo/logserver/LogDispatcher.java | 188 ++++ .../java/com/yahoo/logserver/PluginLoader.java | 22 + .../src/main/java/com/yahoo/logserver/Server.java | 208 +++++ .../main/java/com/yahoo/logserver/demo/.gitignore | 0 .../com/yahoo/logserver/filter/LevelFilter.java | 30 + .../java/com/yahoo/logserver/filter/LogFilter.java | 23 + .../yahoo/logserver/filter/LogFilterManager.java | 99 +++ .../com/yahoo/logserver/filter/MetricsFilter.java | 50 ++ .../com/yahoo/logserver/filter/MuteFilter.java | 30 + .../yahoo/logserver/filter/NoMetricsFilter.java | 21 + .../com/yahoo/logserver/filter/NullFilter.java | 19 + .../yahoo/logserver/formatter/LogFormatter.java | 28 + .../logserver/formatter/LogFormatterManager.java | 70 ++ .../yahoo/logserver/formatter/NullFormatter.java | 29 + .../yahoo/logserver/formatter/TextFormatter.java | 51 ++ .../logserver/handlers/AbstractLogHandler.java | 137 +++ .../yahoo/logserver/handlers/HandlerThread.java | 241 +++++ .../com/yahoo/logserver/handlers/LogHandler.java | 51 ++ .../lasterrorsholder/LastErrorsHolder.java | 186 ++++ .../LastErrorsHolderConnection.java | 141 +++ .../lasterrorsholder/LastErrorsHolderPlugin.java | 51 ++ .../handlers/logmetrics/LogMetricsHandler.java | 233 +++++ .../handlers/logmetrics/LogMetricsPlugin.java | 54 ++ .../handlers/replicator/FormattedBufferCache.java | 78 ++ .../logserver/handlers/replicator/Replicator.java | 130 +++ .../handlers/replicator/ReplicatorConnection.java | 411 +++++++++ .../handlers/replicator/ReplicatorPlugin.java | 58 ++ .../com/yahoo/logserver/net/LogConnection.java | 272 ++++++ .../yahoo/logserver/net/LogConnectionFactory.java | 41 + .../com/yahoo/logserver/net/control/Levels.java | 125 +++ .../com/yahoo/logserver/net/control/State.java | 54 ++ .../yahoo/logserver/testutils/VerifyLogfile.java | 65 ++ .../src/main/java/com/yahoo/plugin/Config.java | 23 + .../src/main/java/com/yahoo/plugin/Plugin.java | 33 + .../com/yahoo/plugin/SystemPropertyConfig.java | 35 + logserver/src/test/files/event.txt.gz | Bin 0 -> 11272 bytes logserver/src/test/files/logEntries.txt | 3 + logserver/src/test/files/value-events.txt | 982 +++++++++++++++++++++ .../java/com/yahoo/logserver/FlusherTestCase.java | 25 + .../java/com/yahoo/logserver/ServerTestCase.java | 55 ++ .../filter/test/LogFilterManagerTestCase.java | 43 + .../filter/test/MetricsFilterTestCase.java | 45 + .../filter/test/NoMetricsFilterTestCase.java | 56 ++ .../test/LogFormatterManagerTestCase.java | 36 + .../formatter/test/NullFormatterTestCase.java | 30 + .../formatter/test/TextFormatterTestCase.java | 46 + .../logserver/handlers/HandlerThreadTestCase.java | 96 ++ .../lasterrorsholder/LastErrorsHolderTestCase.java | 115 +++ .../logmetrics/test/LogMetricsTestCase.java | 122 +++ .../replicator/FormattedBufferCacheTestCase.java | 43 + .../replicator/ReplicatorPluginTestCase.java | 43 + .../handlers/replicator/ReplicatorTestCase.java | 134 +++ .../com/yahoo/logserver/handlers/status/.gitignore | 0 .../com/yahoo/logserver/httphandlers/.gitignore | 0 .../logserver/net/control/test/LevelsTestCase.java | 92 ++ .../logserver/net/test/LogConnectionTestCase.java | 211 +++++ .../logserver/test/LogDispatcherTestCase.java | 131 +++ .../yahoo/logserver/test/LogMessageTestCase.java | 64 ++ .../com/yahoo/logserver/test/MockLogEntries.java | 53 ++ 66 files changed, 6025 insertions(+) create mode 100644 logserver/.gitignore create mode 100644 logserver/OWNERS create mode 100755 logserver/bin/logserver-start.sh create mode 100644 logserver/pom.xml create mode 100644 logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/Flusher.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/PluginLoader.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/Server.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/demo/.gitignore create mode 100644 logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/net/control/State.java create mode 100644 logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java create mode 100644 logserver/src/main/java/com/yahoo/plugin/Config.java create mode 100644 logserver/src/main/java/com/yahoo/plugin/Plugin.java create mode 100644 logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java create mode 100644 logserver/src/test/files/event.txt.gz create mode 100644 logserver/src/test/files/logEntries.txt create mode 100644 logserver/src/test/files/value-events.txt create mode 100644 logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/handlers/status/.gitignore create mode 100644 logserver/src/test/java/com/yahoo/logserver/httphandlers/.gitignore create mode 100644 logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java create mode 100644 logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java (limited to 'logserver') diff --git a/logserver/.gitignore b/logserver/.gitignore new file mode 100644 index 00000000000..3c9aa00964f --- /dev/null +++ b/logserver/.gitignore @@ -0,0 +1,16 @@ +.classpath +.git +.project +archive +build +classes +javamake.pdb +log +logarchive +logserver.iws +plugins +staging +target +testLogs +vespatest_dump +/pom.xml.build diff --git a/logserver/OWNERS b/logserver/OWNERS new file mode 100644 index 00000000000..67cd2820bb8 --- /dev/null +++ b/logserver/OWNERS @@ -0,0 +1 @@ +arnej27959 diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh new file mode 100755 index 00000000000..96dac54aafe --- /dev/null +++ b/logserver/bin/logserver-start.sh @@ -0,0 +1,75 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +ROOT=$VESPA_HOME +export ROOT +cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } + +addopts="-server -Xms64m -Xmx256m -XX:MaxDirectMemorySize=76m -XX:MaxJavaStackTraceDepth=-1" + +oomopt="-XX:OnOutOfMemoryError=kill -9 %p" + +jar="-jar $ROOT/lib/jars/logserver-jar-with-dependencies.jar" + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation + +exec java $addopts "$oomopt" "$@" $jar diff --git a/logserver/pom.xml b/logserver/pom.xml new file mode 100644 index 00000000000..dc997db0386 --- /dev/null +++ b/logserver/pom.xml @@ -0,0 +1,87 @@ + + + + 4.0.0 + + com.yahoo.vespa + parent + 6-SNAPSHOT + ../parent/pom.xml + + logserver + jar + 6-SNAPSHOT + logserver + Log server. + + + junit + junit + test + + + org.hamcrest + hamcrest-core + test + + + com.yahoo.vespa + vespajlib + ${project.version} + + + com.yahoo.vespa + vespalog + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + -Xlint:rawtypes + -Xlint:unchecked + -Xlint:deprecation + + + + + maven-assembly-plugin + + + + com.yahoo.logserver.Server + + + + jar-with-dependencies + + + + + make-assembly + package + + + single + + + + + + org.apache.maven.plugins + maven-install-plugin + + true + + + + + diff --git a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java new file mode 100644 index 00000000000..6d215b9e831 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver; + +import com.yahoo.log.LogLevel; +import com.yahoo.plugin.Plugin; +import com.yahoo.plugin.SystemPropertyConfig; + +import java.util.logging.Logger; + +/** + * TODO: describe class + * + * @author Stig Bakken + */ +public abstract class AbstractPluginLoader implements PluginLoader { + private static final Logger log = Logger.getLogger(AbstractPluginLoader.class.getName()); + + public abstract void loadPlugins(); + + protected void loadFromClass (Class pluginClass) { + Plugin plugin; + try { + plugin = (Plugin)pluginClass.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + log.log(LogLevel.ERROR, pluginClass.getName() + ": load failed: " + e); + throw new RuntimeException(e); + } + + String pname = plugin.getPluginName(); + String prefix = Server.APPNAME + "." + pname + "."; + SystemPropertyConfig config = new SystemPropertyConfig(prefix); + String enable = config.get("enable", "true"); + + if (!enable.equals("true")) { + log.log(LogLevel.INFO, pname + ": plugin disabled by config"); + return; + } + + try { + plugin.initPlugin(config); + log.log(LogLevel.DEBUG, pname + ": plugin loaded"); + } catch (Exception e) { + log.log(LogLevel.ERROR, pname + ": init failed", e); + } + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java new file mode 100644 index 00000000000..f2ef60267fe --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver; + +import java.util.logging.Logger; + +import com.yahoo.log.LogLevel; +import com.yahoo.logserver.handlers.archive.ArchiverPlugin; +import com.yahoo.logserver.handlers.lasterrorsholder.LastErrorsHolderPlugin; +import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin; +import com.yahoo.logserver.handlers.replicator.ReplicatorPlugin; + +/** + * Load a set of builtin plugins + * + * @author Stig Bakken + */ +public class BuiltinPluginLoader extends AbstractPluginLoader { + private static final Logger log = Logger.getLogger(BuiltinPluginLoader.class.getName()); + + public void loadPlugins() { + log.log(LogLevel.DEBUG, "starting to load builtin plugins"); + + loadFromClass(ArchiverPlugin.class); + loadFromClass(ReplicatorPlugin.class); + loadFromClass(LogMetricsPlugin.class); + loadFromClass(LastErrorsHolderPlugin.class); + + log.log(LogLevel.DEBUG, "done loading builtin plugins"); + } + +} diff --git a/logserver/src/main/java/com/yahoo/logserver/Flusher.java b/logserver/src/main/java/com/yahoo/logserver/Flusher.java new file mode 100644 index 00000000000..ee2b317f573 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/Flusher.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver; + +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Logger; + +import com.yahoo.log.LogLevel; +import com.yahoo.logserver.handlers.LogHandler; + +/** + * @author Bjorn Borud + */ +public class Flusher extends Thread { + private static final Logger log = Logger.getLogger(Flusher.class.getName()); + private static final Flusher instance; + private static final List> logHandlers = + new ArrayList>(); + + static { + instance = new Flusher(); + instance.start(); + } + + Flusher() { + super("flusher"); + } + + public static synchronized void register(LogHandler logHandler) { + logHandlers.add(new WeakReference(logHandler)); + } + + public synchronized void run() { + try { + while(!isInterrupted()) { + Thread.sleep(2000); + Iterator> it = logHandlers.iterator(); + while (it.hasNext()) { + WeakReference r = it.next(); + LogHandler h = r.get(); + if (h == null) { + it.remove(); + } else { + h.flush(); + } + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Flushing " + h); + } + } + } + } catch (InterruptedException e) { + log.log(LogLevel.WARNING, "flusher was interrupted", e); + } + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java new file mode 100644 index 00000000000..6de34d4a899 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java @@ -0,0 +1,188 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +import com.yahoo.io.SelectLoopHook; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.handlers.LogHandler; + + +/** + * This is the central point from which LogMessage objects are + * propagated throughout the logserver architecture. + * + * + * @author Bjorn Borud + */ +public class LogDispatcher implements LogHandler, SelectLoopHook +{ + private static final Logger log = Logger.getLogger(LogDispatcher.class.getName()); + + private final List handlers = new ArrayList(); + private int messageCount = 0; + private boolean hasBeenShutDown = false; + private boolean batchedMode = false; + private final int batchSize = 5000; + private List currentBatchList; + private int roundCount = 0; + @SuppressWarnings("unused") + private int lastRoundCount = 0; + + public LogDispatcher () { + } + + /** + * Dispatches a message to all the LogHandler instances we've + * got registered. The main entry point for LogMessage instances + * into the log server. + * + * @param msg The LogMessage instance we wish to dispatch to the + * plugins + */ + public synchronized void handle (LogMessage msg) { + if (msg == null) { + throw new NullPointerException("LogMessage was null"); + } + + if (batchedMode) { + addToBatch(msg); + } else { + for (LogHandler h : handlers) { + h.handle(msg); + } + } + messageCount++; + } + + private void addToBatch (LogMessage msg) { + if (currentBatchList == null) { + currentBatchList = new ArrayList(batchSize); + currentBatchList.add(msg); + return; + } + + currentBatchList.add(msg); + + if (currentBatchList.size() == batchSize) { + flushBatch(); + } + } + + private void flushBatch () { + if (currentBatchList == null) { + return; + } + + for (LogHandler ht : handlers) { + ht.handle(currentBatchList); + } + currentBatchList = null; + } + + + public void handle (List messages) { + throw new IllegalStateException("method not supported"); + } + + /** + * Set the batched mode. Note that this should only be set + * at initialization time because it radically changes the + * behavior of the dispatcher. When in batched mode, the + * dispatcher will not enqueue single LogMessage instances + * but lists of same. + */ + public void setBatchedMode (boolean batchedMode) { + this.batchedMode = batchedMode; + } + + public synchronized void flush () { + if (batchedMode) { + flushBatch(); + } + + for (LogHandler h : handlers) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Flushing " + h.toString()); + } + h.flush(); + } + } + + public synchronized void close () { + if (hasBeenShutDown) { + throw new IllegalStateException("Shutdown already in progress"); + } + hasBeenShutDown = true; + + for (LogHandler ht : handlers) { + if (ht instanceof Thread) { + log.fine("Stopping " + ht); + ((Thread)ht).interrupt(); + } + } + handlers.clear(); + + log.log(LogLevel.DEBUG, "Logdispatcher shut down. Handled " + messageCount + " messages"); + } + + /** + * Register handler thread with the dispatcher. If the handler + * thread has already been registered, we log a warning and + * just do nothing. + * + *

+ * If the thread is not alive it will be start()'ed. + * + */ + public synchronized void registerLogHandler (LogHandler ht) { + if (hasBeenShutDown) { + throw new IllegalStateException("Tried to register LogHandler on"+ + " LogDispatcher which was shut down"); + } + + if (handlers.contains(ht)) { + log.warning("LogHandler was already registered: " + ht); + return; + } + handlers.add(ht); + + if ((ht instanceof Thread) && (! ((Thread)ht).isAlive())) { + ((Thread)ht).start(); + } + + log.fine("Added (and possibly started) LogHandler " + ht); + } + + /** + * Make defensive copy and return array of LogHandlers. + */ + public LogHandler[] getLogHandlers () { + LogHandler[] h = new LogHandler[handlers.size()]; + return handlers.toArray(h); + } + + /** + * Return message counter. + * + * @return Returns the number of messages that we have seen. + */ + public synchronized int getMessageCount () { + return messageCount; + } + + /** + * Hook which is called when the select loop has finished. + */ + public void selectLoopHook (boolean before) { + if (batchedMode) { + flushBatch(); + } + + lastRoundCount = messageCount - roundCount; + roundCount = messageCount; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/PluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/PluginLoader.java new file mode 100644 index 00000000000..2357eefb804 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/PluginLoader.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver; + +/** + * This interface specifies an API for implementing logserver plugin + * loaders. A plugin loader has two basic tasks: to load or unload + * all of its knows plugins. In addition, if a plugin loader's + * canReload() method returns true, plugins may be loaded + * again after they are unloaded. + * + *

Plugins loaded through such reload-capable plugin loaders may + * be upgraded without restarting the server. + * + * @author Stig Bakken + */ +public interface PluginLoader +{ + /** + * Load all plugins known to this loader. + */ + public void loadPlugins(); +} diff --git a/logserver/src/main/java/com/yahoo/logserver/Server.java b/logserver/src/main/java/com/yahoo/logserver/Server.java new file mode 100644 index 00000000000..d09ae305dc2 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/Server.java @@ -0,0 +1,208 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver; + +import com.yahoo.io.FatalErrorHandler; +import com.yahoo.io.Listener; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogSetup; +import com.yahoo.log.event.Event; +import com.yahoo.logserver.handlers.HandlerThread; +import com.yahoo.logserver.handlers.LogHandler; +import com.yahoo.logserver.net.LogConnectionFactory; +import com.yahoo.logserver.net.control.Levels; +import com.yahoo.system.CatchSigTerm; +import java.io.IOException; +import java.util.HashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This class implements the log server itself. At present there is + * no runtime configuration; the server starts up, binds port 19081 + * and loads builtin plugins. + * + * @author Bjorn Borud + * @author Stig Bakken + */ + +public class Server implements Runnable { + private final AtomicBoolean signalCaught = new AtomicBoolean(false); + public static final String APPNAME = "logserver"; + private static final Server instance = new Server(); + private static final Logger log = Logger.getLogger(Server.class.getName()); + private static final FatalErrorHandler fatalErrorHandler = + new FatalErrorHandler(); + private static final HashMap handlerThreads = + new HashMap(); + private static final HashMap threadNameForHandler = + new HashMap(); + + static { + LogSetup.initVespaLogging("ADM"); + } + + // the port is a String because we want to use it as the default + // value of a System.getProperty(). + private static final String LISTEN_PORT = "19081"; + + private int listenPort; + private Listener listener; + private final LogDispatcher dispatch; + + private final boolean isInitialized; + + /** + * Server constructor + */ + private Server () { + dispatch = new LogDispatcher(); + dispatch.setBatchedMode(true); + isInitialized = false; + } + + public static Server getInstance () { + return instance; + } + + private HandlerThread getHandlerThread (String threadName) { + threadName += " handler thread"; + HandlerThread ht = handlerThreads.get(threadName); + if (ht == null) { + ht = new HandlerThread(threadName); + handlerThreads.put(threadName, ht); + ht.setFatalErrorHandler(fatalErrorHandler); + dispatch.registerLogHandler(ht); + } + return ht; + } + + public void registerPluginLoader (PluginLoader loader) { + loader.loadPlugins(); + } + + public void registerLogHandler (LogHandler lh, String threadName) { + HandlerThread ht = getHandlerThread(threadName); + ht.registerHandler(lh); + threadNameForHandler.put(lh, threadName); + } + + public void unregisterLogHandler (LogHandler lh) { + String threadName = threadNameForHandler.get(lh); + unregisterLogHandler(lh, threadName); + } + + public void unregisterLogHandler (LogHandler lh, String threadName) { + HandlerThread ht = getHandlerThread(threadName); + ht.unregisterHandler(lh); + threadNameForHandler.remove(lh); + } + + public void registerFlusher (LogHandler lh) { + Flusher.register(lh); + } + + /** Included only for consistency */ + public void unregisterFlusher (LogHandler lh) { + /* NOP */ + } + + /** + * Initialize the server and start up all its plugins, + * + * @param listenPort The port on which the logserver accepts log + * messages. + */ + public void initialize (int listenPort) { + if (isInitialized) { + throw new IllegalStateException(APPNAME + " already initialized"); + } + + this.listenPort = listenPort; + + // plugins + registerPluginLoader(new BuiltinPluginLoader()); + + // main listener + listener = new Listener(APPNAME); + listener.addSelectLoopPostHook(dispatch); + listener.setFatalErrorHandler(fatalErrorHandler); + } + + /** + * Sets up the listen port and starts the Listener. Then waits for + * Listener to exit. + */ + public void run() { + try { + listener.listen(new LogConnectionFactory(dispatch), listenPort); + log.log(LogLevel.CONFIG, APPNAME + ".listenport=" + listenPort); + } catch (IOException e) { + log.log(LogLevel.ERROR, "Unable to initialize", e); + return; + } + + log.fine("Starting listener..."); + listener.start(); + Event.started(APPNAME); + try { + listener.join(); + log.fine("listener thread exited"); + } + catch (InterruptedException e) { + log.log(Level.WARNING, "Server was interrupted", e); + } + } + private void setupSigTermHandler() { + CatchSigTerm.setup(signalCaught); // catch termination signal + } + + private void waitForShutdown() { + synchronized (signalCaught) { + while (!signalCaught.get()) { + try { + signalCaught.wait(); + } catch (InterruptedException e) { + // empty + } + } + } + Event.stopping(APPNAME, "shutdown"); + dispatch.close(); + Event.stopped(APPNAME, 0, 0); + System.exit(0); + } + + public static HashMap threadNameForHandler() { + return threadNameForHandler; + } + + public static void help () { + System.out.println(); + System.out.println("System properties:"); + System.out.println(" - " + APPNAME + ".listenport (" + LISTEN_PORT + ")"); + System.out.println(" - " + APPNAME + ".queue.size (" + + HandlerThread.DEFAULT_QUEUESIZE + + ")"); + System.out.println(" - logserver.default.loglevels (" + + (new Levels()).toString() + + ")"); + System.out.println(); + } + + public static void main (String[] args) { + if (args.length > 0 && "-help".equals(args[0])) { + help(); + System.exit(0); + } + + String portString = System.getProperty(APPNAME + ".listenport", LISTEN_PORT); + Server server = Server.getInstance(); + server.setupSigTermHandler(); + server.initialize(Integer.parseInt(portString)); + + Thread t = new Thread(server, "logserver main"); + t.start(); + server.waitForShutdown(); + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/demo/.gitignore b/logserver/src/main/java/com/yahoo/logserver/demo/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java new file mode 100644 index 00000000000..b229af522ab --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.filter; + +import com.yahoo.log.LogMessage; +import java.util.Set; +import java.util.HashSet; +import java.util.logging.Level; +/** + * + * @author Bjorn Borud + */ +public class LevelFilter implements LogFilter { + private final Set levels = new HashSet(); + + public void addLevel (Level level) { + levels.add(level); + } + + public void removeLevel (Level level) { + levels.remove(level); + } + + public boolean isLoggable (LogMessage msg) { + return levels.contains(msg.getLevel()); + } + + public String description () { + return "Match specific log levels"; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java new file mode 100644 index 00000000000..8c4c3911c9e --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.filter; + +import com.yahoo.log.LogMessage; + +/** + * This interface is analogous to the java.util.logging.Filter + * interface. Classes implementing this interface should be + * stateless/immutable if possible so filters can be + * shared. + * + * @author Bjorn Borud + */ +public interface LogFilter { + /** + * Determine if this log message is loggable. + * + * @param msg The log message + * + */ + public boolean isLoggable (LogMessage msg); + public String description (); +} diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java new file mode 100644 index 00000000000..a68aff86e10 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.filter; + +import java.util.HashMap; +import java.util.Map; + +import com.yahoo.log.LogLevel; + +/** + * The LogFilterManager keeps track of associations between + * LogFilter names and instances, so that access to filters + * is truly global. It also manages the LogFilter namespace + * to ensure that system-defined filters are not tampered with. + * + * @author Bjorn Borud + */ + +public class LogFilterManager { + private static final LogFilterManager instance; + + static { + instance = new LogFilterManager(); + + LevelFilter allEvents = new LevelFilter(); + allEvents.addLevel(LogLevel.EVENT); + instance.addLogFilterInternal("system.allevents", allEvents); + instance.addLogFilterInternal("system.metricsevents", new MetricsFilter()); + instance.addLogFilterInternal("system.nometricsevents", new NoMetricsFilter()); + instance.addLogFilterInternal("system.all", new NullFilter()); + instance.addLogFilterInternal("system.mute", MuteFilter.getInstance()); + } + + private final Map filters = new HashMap(); + + private LogFilterManager () { + } + + /** + * Public interface for adding a name-logfilter mapping. If + * there exists a mapping already the old mapping is replaced + * with the new mapping. + * + * If the name is within the namespace reserved for internal + * built-in filters it will throw an exception + */ + public static void addLogFilter (String name, LogFilter filter) { + if (filter == null) { + throw new NullPointerException("filter cannot be null"); + } + + if (name == null) { + throw new NullPointerException("name cannot be null"); + } + + String n = name.toLowerCase(); + + if (n.startsWith("system.")) { + throw new IllegalArgumentException("'system' namespace is reserved"); + } + + instance.addLogFilterInternal(n, filter); + } + + /** + * LogFilter lookup function + * + * @param name The name of the LogFilter to be looked up. + * @return Returns the LogFilter associated with this name or + * null if not found. + */ + public static LogFilter getLogFilter (String name) { + return instance.filters.get(name); + } + + + /** + * Get the names of the defined filters. + * + * @return Returns an array containing the names of filters that + * have been registered. + * + */ + public static String[] getFilterNames () { + synchronized(instance.filters) { + String[] filterNames= new String[instance.filters.keySet().size()]; + instance.filters.keySet().toArray(filterNames); + return filterNames; + } + } + + /** + * Internal method which takes care of the job of adding + * LogFilter mappings but doesn't perform any of the checks + * performed by the public method for adding mappings. + */ + private void addLogFilterInternal (String name, LogFilter filter) { + filters.put(name, filter); + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java new file mode 100644 index 00000000000..fa626de1d08 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.filter; + +import com.yahoo.log.LogLevel; +import com.yahoo.log.event.Count; +import com.yahoo.log.event.CountGroup; +import com.yahoo.log.event.Event; +import com.yahoo.log.event.Histogram; +import com.yahoo.log.event.MalformedEventException; +import com.yahoo.log.event.Value; +import com.yahoo.log.event.ValueGroup; +import com.yahoo.log.LogMessage; + +/** + * This filter matches events that are used for monitoring, specificly + * the Count and Value events. + * + * @author Bjorn Borud + */ +public class MetricsFilter implements LogFilter { + public boolean isLoggable (LogMessage msg) { + if (msg.getLevel() != LogLevel.EVENT) { + return false; + } + + Event event; + try { + event = msg.getEvent(); + } + catch (MalformedEventException e) { + return false; + } + + // if it is not Count, Value or something which will generate + // Count or Value we don't care + if (! ((event instanceof Count) + || (event instanceof Value) + || (event instanceof Histogram) + || (event instanceof CountGroup) + || (event instanceof ValueGroup))) { + return false; + } + + return true; + } + + public String description () { + return "Match all events representing system metrics (Counts, Values, etc)."; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java new file mode 100644 index 00000000000..a360cba6b8f --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.filter; + +import com.yahoo.log.LogMessage; + +/** + * Filter which always returns false. + * + * @author Bjorn Borud + */ +public class MuteFilter implements LogFilter { + private static final MuteFilter instance = new MuteFilter(); + + /** + * Singleton, private constructor. + */ + private MuteFilter () {} + + public static MuteFilter getInstance() { + return instance; + } + + public boolean isLoggable (LogMessage msg) { + return false; + } + + public String description () { + return "Matches no messages. Mute."; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java new file mode 100644 index 00000000000..5621ea60d0c --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.filter; + +import com.yahoo.log.LogMessage; + +/** + * This filter is the complement of MetricsFilter + * + * @author Bjorn Borud + */ +public class NoMetricsFilter implements LogFilter { + final MetricsFilter filter = new MetricsFilter(); + + public boolean isLoggable (LogMessage msg) { + return (! filter.isLoggable(msg)); + } + + public String description () { + return "Matches all log messages except Count and Value events"; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java new file mode 100644 index 00000000000..a62b2f7171b --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.filter; + +import com.yahoo.log.LogMessage; + +/** + * + * + * @author Bjorn Borud + */ +public class NullFilter implements LogFilter { + public boolean isLoggable (LogMessage msg) { + return true; + } + + public String description () { + return "Match all log messages"; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java new file mode 100644 index 00000000000..ae217634e1c --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.formatter; + +import com.yahoo.log.LogMessage; + +/** + * This interface is analogous to the java.util.logging.Formatter + * interface. Classes implementing this interface should be + * stateless/immutable if possible so formatters can be + * shared. If it does have state it must not prevent + * concurrent use. + * + * @author Bjorn Borud + */ +public interface LogFormatter { + /** + * Format log message as a string. + * + * @param msg The log message + * + */ + public String format (LogMessage msg); + + /** + * Returns a textual description of the formatter + */ + public String description (); +} diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java new file mode 100644 index 00000000000..fb81b35dcf0 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * $Id$ + * + */ + +package com.yahoo.logserver.formatter; + +import java.util.HashMap; +import java.util.Map; + +/** + * This singleton class implements a central registry of LogFormatter + * instances. + * + * @author Bjorn Borud + */ +public class LogFormatterManager { + private static final LogFormatterManager instance; + static { + instance = new LogFormatterManager(); + instance.addLogFormatterInternal("system.textformatter", new TextFormatter()); + instance.addLogFormatterInternal("system.nullformatter", new NullFormatter()); + } + + private final Map logFormatters = + new HashMap(); + + private LogFormatterManager () {} + + /** + * LogFormatter lookup function + * + * @param name The name of the LogFormatter to be looked up. + * @return Returns the LogFormatter associated with this name or + * null if not found. + */ + public static LogFormatter getLogFormatter (String name) { + synchronized(instance.logFormatters) { + return instance.logFormatters.get(name); + } + } + + /** + * Get the names of the defined formatters. + * + * @return Returns an array containing the names of formatters that + * have been registered. + * + */ + public static String[] getFormatterNames () { + synchronized(instance.logFormatters) { + String[] formatterNames = new String[instance.logFormatters.keySet().size()]; + instance.logFormatters.keySet().toArray(formatterNames); + return formatterNames; + } + } + + /** + * Internal method which takes care of the job of adding + * LogFormatter mappings but doesn't perform any of the checks + * performed by the public method for adding mappings. + */ + private void addLogFormatterInternal (String name, LogFormatter logFormatter) { + synchronized(logFormatters) { + logFormatters.put(name, logFormatter); + } + } + +} diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java new file mode 100644 index 00000000000..646ab0ce423 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * $Id$ + * + */ + +package com.yahoo.logserver.formatter; + +import com.yahoo.log.LogMessage; + +/** + * This formatter doesn't really format anything. It just + * calls the LogMessage toString() method. This is kind of + * pointless and silly, but we include it for symmetry... + * or completeness....or...whatever. + * + * @author Bjorn Borud + */ +public class NullFormatter implements LogFormatter { + + public String format (LogMessage msg) { + return msg.toString(); + } + + public String description() { + return "Format message in native VESPA format"; + } + +} diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java new file mode 100644 index 00000000000..d94dcceaf29 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * $Id$ + * + */ + +package com.yahoo.logserver.formatter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.TimeZone; + +import com.yahoo.log.LogMessage; + +/** + * Creates human-readable text representation of log message. + * + * @author Bjorn Borud + */ +public class TextFormatter implements LogFormatter { + static final SimpleDateFormat dateFormat; + + static { + dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + public String format (LogMessage msg) { + StringBuffer sbuf = new StringBuffer(150); + sbuf.append(dateFormat.format(new Date(msg.getTime()))) + .append(" ") + .append(msg.getHost()) + .append(" ") + .append(msg.getThreadProcess()) + .append(" ") + .append(msg.getService()) + .append(" ") + .append(msg.getComponent()) + .append(" ") + .append(msg.getLevel().toString()) + .append(" ") + .append(msg.getPayload()) + .append("\n"); + + return sbuf.toString(); + } + + public String description() { + return "Format log-message as human readable text"; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java new file mode 100644 index 00000000000..f498568e7f2 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java @@ -0,0 +1,137 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers; + +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.List; + +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.filter.LogFilter; + +/** + * This abstract class is the one you would usually want to + * extend when you are writing a LogHandler since it takes care + * of quite a bit of tedious work for you (log message counting, + * handling of lists of messages versus single instances etc). + * + * @author Bjorn Borud + */ +public abstract class AbstractLogHandler implements LogHandler { + private long count = 0; + private long filtered = 0; + private LogFilter filter = null; + private String name; + + /** + * This is the entry point for each log handler. Takes care + * of calling the actual doHandle() method. Provided to make + * it possible to extend what a handler does in a uniform way. + * + * @param msg The message we are about to handle + */ + public final void handle (LogMessage msg) { + if ((filter != null) && (! filter.isLoggable(msg))) { + filtered++; + return; + } + + if (doHandle(msg)) { + count++; + } + } + + /** + * Handle a list of LogMessage instances + * + * @param messages List of LogMessage instances. + */ + public final void handle (List messages) { + Iterator it = messages.iterator(); + while (it.hasNext()) { + handle(it.next()); + } + } + + /** + * Set LogFilter for this LogHandler. If the LogFilter is + * null, filtering has in effect been turned + * off. + * + * @param filter The filter to be used for this handler + * + */ + public void setLogFilter (LogFilter filter) { + this.filter = filter; + } + + /** + * @return Returns the log filter for this handler or + * null if no filter is in effect. + */ + public LogFilter getLogFilter () { + return filter; + } + + /** + * Returns the internal counter which keeps track of the number + * of times doHandle has been called. + * + * @return Returns the number of times doHandle has been called. + */ + public final long getCount () { + return count; + } + + public String getName () { + if (name == null) { + String n = this.getClass().getName(); + int x = n.lastIndexOf('.'); + if (x != -1) { + n = n.substring(x+1); + } + name = n; + } + return name; + } + + public void setName (String name) { + this.name = name; + } + + /** + * The method which actually handles the log message and + * does something to it. This is the one you wish to + * override when you write a new handler. + * + *

+ * + * If your handle method is slow you should document this fact + * so that decisions can be made with regard to configuration. + * + * + * @param msg The LogMessage we are about to handle + * @return Returns true if the message was + * handled and false if it was ignored. + */ + public abstract boolean doHandle (LogMessage msg); + + /** + * Flush LogMessages. + */ + public abstract void flush (); + + /** + * Close this loghandler. After a loghandler is closed calling + * the #handle() has undefined behavior, but it should be assumed + * that log messages will be silently dropped. + * + *

+ * #close() usually implies #flush() but don't bet on it. + */ + public abstract void close (); + + /** + * Force implementation of (hopefully meaningful) toString() + */ + public abstract String toString(); +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java new file mode 100644 index 00000000000..8af5202739b --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java @@ -0,0 +1,241 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Logger; + +import com.yahoo.io.FatalErrorHandler; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * This handler implements a dispatcher which runs in its own + * thread. The purpose of this handler is to isolate execution + * of handlers from the main server IO threads. + * + * @author Bjorn Borud + */ + +public class HandlerThread extends Thread implements LogHandler +{ + private static final Logger log = Logger.getLogger(HandlerThread.class.getName()); + + // default queue size is 200 + public static final int DEFAULT_QUEUESIZE = 200; + private static int queueSize = DEFAULT_QUEUESIZE; + + private FatalErrorHandler fatalErrorHandler; + + // set other queue size if specified + static { + String queueSizeStr = System.getProperty("logserver.queue.size"); + if (queueSizeStr != null) { + queueSize = Integer.parseInt(queueSizeStr); + + // should never be smaller than 50 + if (queueSize < 50) { + queueSize = 50; + } + + log.info("set queue size to " + queueSize); + } + } + + private static class ItemOrList { + final LogMessage item; + final List list; + ItemOrList(LogMessage i) { + this.item = i; + this.list = null; + } + ItemOrList(List l) { + this.item = null; + this.list = l; + } + public String toString() { + return "item="+item+", list="+list; + } + } + + private final BlockingQueue queue; + private final List handlers = new ArrayList(); + private long count; + @SuppressWarnings("unused") + private long droppedCount = 0; + @SuppressWarnings("unused") + private boolean queueWasFull = false; + @SuppressWarnings("unused") + private long lastDropLogMessage = 0; + @SuppressWarnings("unused") + private long lastAcceptingLogMessage = 0; + + public HandlerThread (String name) { + super(name); + queue = new LinkedBlockingQueue<>(queueSize); + log.log(LogLevel.CONFIG, "logserver.queue.size=" + queueSize); + } + + /** + * Register a handler for fatal errors. + * + * @param f The FatalErrorHandler instance to be registered + */ + public synchronized void setFatalErrorHandler (FatalErrorHandler f) { + fatalErrorHandler = f; + } + + /** + * Called by the LogDispatch to put a LogMessage onto the Queue + * + * @param message The LogMessage we wish to dispatch to this + * handler thread. + */ + public void handle (LogMessage message) { + handleInternal(new ItemOrList(message)); + } + + /** + * Called by the LogDispatch to put a list of LogMessage + * instances onto the Queue. + */ + public void handle (List messages) { + handleInternal(new ItemOrList(messages)); + } + + private void handleInternal (ItemOrList o) { + boolean done = false; + while (! done) { + try { + queue.put(o); + done = true; + } + catch (InterruptedException e) { + // NOP + } + } + } + + public void flush () { + Iterator it = handlers.iterator(); + while (it.hasNext()) { + LogHandler handler = (LogHandler)it.next(); + handler.flush(); + } + } + + public void close () { + Iterator it = handlers.iterator(); + while (it.hasNext()) { + LogHandler handler = (LogHandler)it.next(); + handler.close(); + } + } + + public long getCount () { + return count; + } + + /** + * Register a LogHandler + */ + public synchronized void registerHandler (LogHandler handler) { + log.fine("Registering handler " + handler); + handlers.add(handler); + } + + /** + * Unregister a Loghandler + */ + public synchronized void unregisterHandler (LogHandler handler) { + int idx; + while ((idx = handlers.indexOf(handler)) != -1) { + try { + handlers.remove(idx); + } catch (IndexOutOfBoundsException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Return an array of the registered handlers. + * + * @return Returns an array of the handlers registered + */ + public LogHandler[] getHandlers () { + LogHandler[] h = new LogHandler[handlers.size()]; + return handlers.toArray(h); + } + + + /** + * Return the underlying queue used to send LogMessage instances + * to this handler thread. + */ + public BlockingQueue getQueue () { + return queue; + } + + /** + * Consume messages from the incoming queue and hand + * them off to the handlers. + */ + public void run () { + if (queue == null) { + throw new NullPointerException("channel is not allowed to be null"); + } + + // TODO: Make the legmessage elements some kind of composite structure to handle both individual messages and lists uniformly. + List drainList = new ArrayList(queue.size() + 1); + try { + for (;;) { + drainList.clear(); + // block in take(), then see if there is more + // to be had with drainTo() + drainList.add(queue.take()); + queue.drainTo(drainList); + + for (ItemOrList o : drainList) { + // we can get two types of elements here: single log + // messages or lists of log messages, so we need to + // handle them accordingly. + + if (o.item != null) { + for (LogHandler handler : handlers) { + handler.handle(o.item); + } + } else if (o.list != null) { + for (LogHandler handler : handlers) { + handler.handle(o.list); + } + } else { + throw new IllegalArgumentException("not LogMessage or List: " + o); + } + count++; + } + } + } catch (InterruptedException e) { + // NOP + } catch (Throwable t) { + if (fatalErrorHandler != null) { + fatalErrorHandler.handle(t, null); + } + } finally { + log.fine("Handler thread " + + getName() + + " exiting, removing handlers"); + for (LogHandler handler : handlers) { + log.fine("Removing handler " + handler); + handler.close(); + } + handlers.clear(); + log.fine("Handler thread " + getName() + " done"); + } + + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java new file mode 100644 index 00000000000..6fb009fcda2 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers; + +import com.yahoo.log.LogMessage; +import java.util.List; + +/** + * The LogHandler interface defines the interface used for all + * parts of the logserver which consume log messages. + * + * @author Bjorn Borud + */ +public interface LogHandler { + /** + * This is the entry point for the log handling. This method + * should return as quickly as possible in implementations + * so if you need to initiate time consuming processing you + * should look into some design alternatives. + * + * @param msg The log message + */ + public void handle (LogMessage msg); + + /** + * Instead of taking a single log message, this method can take + * a List of them. The List abstraction was chosen because the + * order needs to be preserved. + * + * @param messages a List containing zero or more LogMessage + * instances. + */ + public void handle (List messages); + + /** + * Any log messages received so far should be dealt with + * before this method returns -- within reason ,of course. + * (Within reason is loosely defined to be 2-5 seconds) + */ + public void flush (); + + /** + * Signals that we want to end logging and should close down the + * unerlying logging mechanism -- whatever this maps to + * semantically for the underlying implementation. After this + * method has been called it is considered an error to submit more + * log messages to the handle() methods and an implementation + * may elect to throw runtime exceptions. + * + */ + public void close (); +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java new file mode 100644 index 00000000000..93552bf9f00 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.lasterrorsholder; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.io.Connection; +import com.yahoo.io.ConnectionFactory; +import com.yahoo.io.Listener; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.handlers.AbstractLogHandler; + +import java.io.IOException; +import java.io.StringWriter; +import java.nio.channels.SocketChannel; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +/** + * The LastErrorsHolder handler is used for holding the last n + * messages at level error or higher. Connecting to this handler + * will return a Json object with the last errors (default is last 100 errors) + * + * @author musum + */ +public class LastErrorsHolder extends AbstractLogHandler implements ConnectionFactory { + private static final Logger log = Logger.getLogger(LastErrorsHolder.class.getName()); + private static final int maxNumErrors = 100; + private final Object lock = new Object(); + + private int port; + private Listener listener; + + private final ArrayList errors = new ArrayList<>(); + private int numberOfErrors = 0; + + /** + * @param port The port to which this handler listens to. + */ + public LastErrorsHolder(int port) throws IOException { + this.port = port; + listen(port); + } + + public void listen(int port) throws IOException { + if (listener != null) { + throw new IllegalStateException("already listening to port " + this.port); + } + listener = new Listener("last-errors-holder"); + listener.listen(this, port); + listener.start(); + log.log(LogLevel.CONFIG, "port=" + port); + } + + public boolean doHandle(LogMessage msg) { + if (msg.getLevel().equals(LogLevel.ERROR) || msg.getLevel().equals(LogLevel.FATAL)) { + synchronized (lock) { + numberOfErrors++; + if (errors.size() < maxNumErrors) { + errors.add(msg); + } else if (numberOfErrors == maxNumErrors) { + log.log(LogLevel.DEBUG, String.format("Not storing errors, have reached maximum number of errors: %d, total number of errors received: %d", + maxNumErrors, numberOfErrors)); + } + } + } + return true; + } + + public void close() { + try { + listener.interrupt(); + listener.join(); + log.log(LogLevel.DEBUG, "listener stopped"); + } catch (InterruptedException e) { + log.log(LogLevel.WARNING, "listener was interrupted", e); + } + } + + public void flush() { + } + + /** + * Factory method for creating new connections. Since we just return a result + * when client connection happens, we also write the result here + * + * @param socket The new SocketChannel + * @param listener The Listener instance we want to use + */ + public Connection newConnection(SocketChannel socket, Listener listener) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "New last-errors-holder connection: " + socket); + } + LastErrorsHolderConnection connection = new LastErrorsHolderConnection(socket); + synchronized (lock) { + Messages messages = new Messages(); + for (LogMessage error : errors) { + messages.addMessage( + new Message(error.getTime()/1000, + error.getHost(), + error.getService(), + error.getLevel().getName(), + error.getPayload())); + } + messages.setNumberOfErrors(numberOfErrors); + + try { + ObjectMapper mapper = new ObjectMapper(); + StringWriter stringWriter = new StringWriter(); + mapper.writeValue(stringWriter, messages); + connection.enqueue(StandardCharsets.UTF_8.encode(stringWriter.toString())); + } catch (IOException e) { + log.log(LogLevel.WARNING, "Could not enqueue log message", e); + } + + errors.clear(); + numberOfErrors = 0; + } + + return connection; + } + + public String toString() { + return LastErrorsHolder.class.getName(); + } + + + static class Messages { + private final List messages = new ArrayList<>(); + private long errorCount = 0; // There might be more errors than number of messages + + void addMessage(Message message) { + messages.add(message); + } + + void setNumberOfErrors(long errorCount) { + this.errorCount = errorCount; + } + + public List getMessages() { + return messages; + } + + public long getErrorCount() { + return errorCount; + } + } + + static class Message { + private final long time; + private final String hostname; + private final String service; + private final String logLevel; + private final String message; + + Message(long time, String hostname, String service, String logLevel, String message) { + this.time = time; + this.hostname = hostname; + this.service = service; + this.logLevel = logLevel; + this.message = message; + } + + public long getTime() { + return time; + } + + public String getMessage() { + return message; + } + + public String getLogLevel() { + return logLevel; + } + + public String getHostname() { + return hostname; + } + + public String getService() { + return service; + } + } + +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java new file mode 100644 index 00000000000..0dc89e95d11 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.lasterrorsholder; + +import com.yahoo.io.Connection; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.filter.LogFilter; +import com.yahoo.logserver.filter.LogFilterManager; +import com.yahoo.logserver.formatter.LogFormatter; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.SocketChannel; +import java.util.logging.Logger; + +/** + * LastErrorsHandler client connection. + * + * @author musum + */ +public class LastErrorsHolderConnection implements Connection, LogFilter { + private static final Logger log = Logger.getLogger(LastErrorsHolderConnection.class.getName()); + + private final SocketChannel socket; + private ByteBuffer writeBuffer; + private final ByteBuffer readBuffer = ByteBuffer.allocate(4096); + private LogFilter filter = null; + protected LogFormatter formatter = null; + private static final String filterName = "system.mute"; + + /** + * Constructs a LastErrorsHolderConnection. Note that initially the + * filter of this connection is set to MuteFilter, which mutes + * all log messages. + */ + public LastErrorsHolderConnection(SocketChannel socket) { + this.socket = socket; + this.filter = LogFilterManager.getLogFilter(filterName); + } + + /** + * Check if the message is wanted by this particular replicator + * connection. The reason we provide this method is because we + * want to be able to determine if a message is wanted by any + * client before committing resources to creating a ByteBuffer to + * serialize it into. + * + * @param msg The log message offered + */ + public boolean isLoggable(LogMessage msg) { + if (filter == null) { + return true; + } + return filter.isLoggable(msg); + } + + /** + * Return the description of the currently active filter. + */ + public String description() { + if (filter == null) { + return "No filter defined"; + } + return filter.description(); + } + + + /** + * Enqueues a ByteBuffer containing the message destined + * for the client. + * + * @param buffer the ByteBuffer into which the log message is + * serialized. + */ + public synchronized void enqueue(ByteBuffer buffer) throws IOException { + writeBuffer = buffer; + write(); + } + + public void read() throws IOException { + if (!readBuffer.hasRemaining()) { + log.warning("Log message too long. Message exceeds " + + readBuffer.capacity() + + " bytes. Connection dropped."); + close(); + return; + } + + + int ret = socket.read(readBuffer); + if (ret == -1) { + close(); + return; + } + + if (ret == 0) { + if (log.isLoggable(LogLevel.INFO)) { + log.log(LogLevel.INFO, "zero byte read occurred"); + } + } + + readBuffer.flip(); + } + + public synchronized void write() throws IOException { + if (!socket.isOpen()) { + close(); + } + do { + try { + socket.write(writeBuffer); + } catch (IOException e) { + log.log(LogLevel.WARNING, "Error writing", e); + close(); + return; + } + } while (writeBuffer.hasRemaining()); + } + + public synchronized void close() throws IOException { + socket.close(); + writeBuffer = null; + } + + public int selectOps() { + return SelectionKey.OP_READ; + } + + public SocketChannel socketChannel() { + return socket; + } + + public void connect() { + } + + void setFilter(LogFilter filter) { + this.filter = filter; + } +} + diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java new file mode 100644 index 00000000000..7cfdbe2b578 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.lasterrorsholder; + +import com.yahoo.log.LogLevel; +import com.yahoo.logserver.Server; +import com.yahoo.plugin.Config; +import com.yahoo.plugin.Plugin; + +import java.io.IOException; +import java.util.logging.Logger; + +public class LastErrorsHolderPlugin implements Plugin { + private static final String DEFAULT_PORT = "19082"; + private static final Logger log = Logger.getLogger(LastErrorsHolderPlugin.class.getName()); + private LastErrorsHolder lastErrorsHolder; + private final Server server = Server.getInstance(); + + public String getPluginName() { + return "last-errors-holder"; + } + + /** + * Initialize the plugin + */ + public void initPlugin(Config config) { + if (lastErrorsHolder != null) { + throw new IllegalStateException("plugin already initialized: " + getPluginName()); + } + int listenPort = config.getInt("port", DEFAULT_PORT); + String threadName = config.get("thread", getPluginName()); + try { + lastErrorsHolder = new LastErrorsHolder(listenPort); + } catch (IOException e) { + log.log(LogLevel.WARNING, "init failed: " + e); + return; + } + server.registerLogHandler(lastErrorsHolder, threadName); + } + + /** + * Shut down the plugin. + */ + public void shutdownPlugin() { + + if (lastErrorsHolder == null) { + throw new IllegalStateException("plugin not initialized: " + getPluginName()); + } + server.unregisterLogHandler(lastErrorsHolder); + lastErrorsHolder = null; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java new file mode 100644 index 00000000000..fee835b8189 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java @@ -0,0 +1,233 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * $Id$ + */ +package com.yahoo.logserver.handlers.logmetrics; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.Map; +import java.util.TreeMap; +import java.util.ArrayList; +import java.util.logging.Logger; +import java.util.logging.Level; + +import com.yahoo.log.LogLevel; +import com.yahoo.log.event.Event; +import com.yahoo.logserver.filter.LevelFilter; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.handlers.AbstractLogHandler; + +/** + * The LogMetricsHandler stores a count of the number of log messages + * per level per host and sends an event count for this five minutes. + * + * + * @author Harald Musum + */ +public class LogMetricsHandler extends AbstractLogHandler +{ + public static final long EVENTINTERVAL = 5 * 60; // in seconds + + private static final Logger log = + Logger.getLogger(LogMetricsHandler.class.getName()); + + // A list of log metrics per host and per log level + private final List logMetrics = new ArrayList(); + + // The log levels that are handled by this plugin + private static final Level[] levels = {LogLevel.INFO, + LogLevel.WARNING, + LogLevel.SEVERE, + LogLevel.ERROR, + LogLevel.FATAL}; + + + /** + * Constructor sets a default log filter ignoring the config, + * debug and spam levels. + */ + public LogMetricsHandler() { + LevelFilter filter = new LevelFilter(); + + for (Level level : Arrays.asList(levels)) { + filter.addLevel(level); + } + + setLogFilter(filter); + + // Start thread that sends events. + EventGenerator eventThread = new EventGenerator(); + new Thread(eventThread).start(); + } + + public boolean doHandle(LogMessage message) { + String host = message.getHost(); + Level logLevel = message.getLevel(); + + boolean found = false; + if (logMetrics.size() > 0) { + LevelCount count; + // Loop through the list logMetrics and check if there + // exists an element with the same host and level. + for (int i = 0; i < logMetrics.size(); i++) { + count = logMetrics.get(i); + if (count.getHost().equals(host) && + count.getLevel().getName().equals(logLevel.getName())) { + count.addCount(1); + found = true; + break; + } + } + } + + // There is no element in logMetrics with the same host and + // level as in the message, so create a new object and add it + // to the list. + if (!found) { + for (Level level : Arrays.asList(levels)) { + LevelCount levelCount; + if (level.getName().equals(logLevel.getName())) { + levelCount = new LevelCount(host, + level, + 1); + } else { + levelCount = new LevelCount(host, + level, + 0); + + } + logMetrics.add(levelCount); + } + } + return true; + } + + /** + * + * Create event count for each log level and report it. For now we + * add up the numbers for all host on each level and report that. + * + */ + private void sendEvents() { + Map levelCount = getMetricsPerLevel(); + for (Map.Entry entry : levelCount.entrySet()) { + String key = entry.getKey(); + Long count = entry.getValue(); + Event.count("log_message." + key.toLowerCase(), count.longValue()); + } + } + + public void flush() {} + + public void close() {} + + public String toString() { + return LogMetricsHandler.class.getName(); + } + + /** + * Returns the total number of log messages processed by this + * plugin. + * + * @return A count of log messages + */ + public long getMetricsCount() { + long count = 0; + for (LevelCount levelCount : logMetrics) { + count = count + levelCount.getCount(); + } + return count; + } + + /** + * Returns a Map of log level counts (level is key and count is + * value). + * + * @return A Map of log level counts + */ + public Map getMetricsPerLevel() { + Map levelCounts = new TreeMap(); + // Loop through all levels summing the count for all hosts. + for (Level level : Arrays.asList(levels)) { + String levelName = level.getName(); + long count = 0; + for (LevelCount levelCount : logMetrics) { + if (levelName.equals(levelCount.getLevel().getName())) { + count += levelCount.getCount(); + } + } + levelCounts.put(levelName, count); + } + return levelCounts; + } + + /** + * The LevelCount class represents the number (count) of log + * messages with the same log level for a host. + * + */ + private class LevelCount { + private final String host; + private final Level level; + private long count; + + LevelCount(String host, Level level, long count) { + this.host = host; + this.level = level; + this.count = count; + } + + LevelCount(String host, Level level) { + this(host, level, 0); + } + + Level getLevel() { + return level; + } + + String getHost() { + return host; + } + + long getCount() { + return count; + } + + void setCount(long count) { + this.count = count; + } + + void addCount(long add) { + count += add; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Host=" + host + ", level = " + level.getName() + + ",count=" + count); + return sb.toString(); + } + } + + /** + * Implements a thread that sends events every EVENTINTERVAL + * seconds. + * + */ + private class EventGenerator implements Runnable { + public void run() { + // Send events every EVENTINTERVAL seconds + while (true) { + try { + Thread.sleep(EVENTINTERVAL * 1000); + } catch (InterruptedException e) { + log.log(LogLevel.WARNING, e.getMessage()); + } + sendEvents(); + } + } + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java new file mode 100644 index 00000000000..204e35d52bb --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * $Id$ + */ +package com.yahoo.logserver.handlers.logmetrics; + +import java.util.logging.Logger; + +import com.yahoo.logserver.Server; +import com.yahoo.plugin.Config; +import com.yahoo.plugin.Plugin; + + +public class LogMetricsPlugin implements Plugin +{ + private static final Logger log = + Logger.getLogger(LogMetricsPlugin.class.getName()); + private LogMetricsHandler logMetricsHandler; + private final Server server = Server.getInstance(); + + public String getPluginName() { + return "logmetrics"; + } + + /** Initialize the logmetrics plugin + * + * @param config Plugin config object, keys used: + * thread - name of handler thread this plugin runs in + * + */ + public void initPlugin(Config config) { + if (logMetricsHandler != null) { + log.finer("LogMetricsPlugin doubly initialized"); + throw new IllegalStateException( + "plugin already initialized: " + getPluginName()); + } + String threadName = config.get("thread", getPluginName()); + logMetricsHandler = new LogMetricsHandler(); + server.registerLogHandler(logMetricsHandler, threadName); + } + + /** + * Shut down the logmetrics plugin. + */ + public void shutdownPlugin() { + if (logMetricsHandler == null) { + log.finer("LogMetricsPlugin shutdown before initialize"); + throw new IllegalStateException("plugin not initialized: " + + getPluginName()); + } + server.unregisterLogHandler(logMetricsHandler); + logMetricsHandler = null; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java new file mode 100644 index 00000000000..f20adbe3c91 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.replicator; + +import java.util.Map; +import java.util.IdentityHashMap; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.formatter.LogFormatter; +import com.yahoo.logserver.formatter.LogFormatterManager; + +/** + * This class is used to cache log messages that have been + * formatted into ByteBuffers. The purpose of this class + * is to make it easier to support multiple message formats while + * still ensuring we don't format more messages than we strictly need + * to and that we don't keep around more buffers that we ought to. + * + *

+ * This is not a general purpose class, I think, so please + * refer to the source code of the Replicator class for + * information on how to use this. + * + *

+ * This class is not threadsafe. + * + * @author Bjorn Borud + */ +public class FormattedBufferCache { + // the documentation says " All of the methods defined in this + // class are safe for use by multiple concurrent threads." so + // we have only one instance of the charset for this class. + // + static private final Charset charset = Charset.forName("utf-8"); + + private final IdentityHashMap buffers; + + public FormattedBufferCache () { + // hope this is a good hash size + int initialSize = LogFormatterManager.getFormatterNames().length * 2; + buffers = new IdentityHashMap(initialSize); + } + + /** + * Return a ByteBuffer slice of a buffer containing the + * LogMessage formatted by the LogFormatter. If one didn't + * exist in the cache from before, it will after this + * method returns. + * + * @param msg The log message you wish to format + * @param formatter The log formatter you wish to use for formatting + * @return Returns a ByteBuffer slice + */ + public ByteBuffer getFormatted (LogMessage msg, LogFormatter formatter) { + ByteBuffer bb = buffers.get(formatter); + if (bb == null) { + bb = charset.encode(formatter.format(msg)); + buffers.put(formatter, bb); + } + return bb.slice(); + } + + /** + * After we're done distributing the log message to all the + * clients we clear the cache so we are ready for the next + * message. + */ + public void reset () { + buffers.clear(); + } + + /** + * This is here for test purposes. Don't get any bright ideas. + */ + public Map getUnderlyingMapOnlyForTesting() { + return buffers; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java new file mode 100644 index 00000000000..f8dc700931f --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java @@ -0,0 +1,130 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.replicator; + +import java.io.IOException; +import java.nio.channels.SocketChannel; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; +import com.yahoo.io.Connection; +import com.yahoo.io.ConnectionFactory; +import com.yahoo.io.Listener; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.handlers.AbstractLogHandler; + +/** + * The Replicator plugin is used for replicating log messages sent + * to the logserver. + * + *

+ * Per default the replicator will start dropping messages enqueued + * to a client if the outbound message queue reaches 5000 messages. + * This limit can be configured by setting the system property + * logserver.replicator.maxqueuelength to the desired + * value. + * + * @author Bjorn Borud + */ +public class Replicator extends AbstractLogHandler +implements ConnectionFactory +{ + private static final Logger log = Logger.getLogger(Replicator.class.getName()); + + private int port; + private Listener listener; + private final Set connections = new HashSet(); + private final FormattedBufferCache bufferCache = new FormattedBufferCache(); + + /** + * @param port The port to which the replicator listens. + * + */ + public Replicator (int port) throws IOException { + this.port = port; + listen(port); + } + + public Replicator () { + } + + public void listen (int port) throws IOException { + if (listener != null) { + throw new IllegalStateException("already listening to port " + this.port); + } + listener = new Listener("replicator"); + listener.listen(this, port); + listener.start(); + log.log(LogLevel.CONFIG, "port=" + port); + } + + public synchronized boolean doHandle (LogMessage msg) { + boolean logged = false; + bufferCache.reset(); + for (ReplicatorConnection c : connections) { + try { + if (c.isLoggable(msg)) { + c.enqueue(bufferCache.getFormatted(msg, c.formatter)); + logged = true; + } + } + catch (IOException e) { + log.log(LogLevel.DEBUG, "Writing failed", e); + } + } + return logged; + } + + public void close() { + // kill the listener thread, then wait for it to + // shut down. + try { + listener.interrupt(); + listener.join(); + log.log(LogLevel.DEBUG, "Replicator listener stopped"); + } + catch (InterruptedException e) { + log.log(LogLevel.WARNING, + "Replicator listener was interrupted", + e); + } + } + + /** + * Currently a NOP, but we might want to have some best-effort + * mechanism for trying to flush all connections within some + * time-frame. + */ + public void flush() {} + + /** + * Factory method for wrapping new connections in the proper + * (Replicator)Connection objects. + * + * @param socket The new SocketChannel + * @param listener The Listener instance we want to use + */ + public synchronized Connection newConnection (SocketChannel socket, + Listener listener) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.fine("New replicator connection: " + socket); + } + ReplicatorConnection n = + new ReplicatorConnection(socket, listener, this); + connections.add(n); + return n; + } + + /** + * Removes a ReplicatorConnection from the set of active + * connections. + */ + protected synchronized void deRegisterConnection (ReplicatorConnection conn) { + connections.remove(conn); + } + + public String toString () { + return Replicator.class.getName(); + } + +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java new file mode 100644 index 00000000000..2978f751665 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java @@ -0,0 +1,411 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.replicator; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.SocketChannel; +import java.util.LinkedList; +import java.util.StringTokenizer; +import java.util.logging.Logger; + +import com.yahoo.io.Connection; +import com.yahoo.io.IOUtils; +import com.yahoo.io.Listener; +import com.yahoo.io.ReadLine; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.filter.LogFilter; +import com.yahoo.logserver.filter.LogFilterManager; +import com.yahoo.logserver.formatter.LogFormatter; +import com.yahoo.logserver.formatter.LogFormatterManager; + +/** + * Replication client connection. + * + * @author Bjorn Borud + */ +public class ReplicatorConnection implements Connection, LogFilter +{ + private static final Logger log + = Logger.getLogger(ReplicatorConnection.class.getName()); + + /** The maximum number of queued messages before we start dropping */ + private static final int maxQueueLength; + /** The maximum number of times we go over maxQueueLength before we log a warning */ + private static final int maxRetriesBeforeWarning = 10; + /** Count of how many times we have received a message while the queue is full */ + private static int queueFullCount = 0; + + static { + String maxQueue = System.getProperty("logserver.replicator.maxqueuelength", + "5000"); + maxQueueLength = Integer.parseInt(maxQueue); + } + + private final SocketChannel socket; + private final String remoteHost; + private final Listener listener; + private final Replicator replicator; + private final LinkedList writeBufferList = new LinkedList(); + private ByteBuffer writeBuffer; + private final ByteBuffer readBuffer = ByteBuffer.allocate(4096); + private LogFilter filter = null; + protected LogFormatter formatter = null; + private String filterName = "system.mute"; + private String formatterName = "system.nullformatter"; + private boolean droppingMode = false; + private int numHandled = 0; + private int numQueued = 0; + private int numDropped = 0; + private long totalBytesWritten = 0; + + /** + * Constructs a ReplicatorConnection. Note that initially the + * filter of this connection is set to MuteFilter, which mutes + * all log messages. + * + */ + public ReplicatorConnection (SocketChannel socket, + Listener listener, + Replicator replicator) { + this.socket = socket; + this.listener = listener; + this.replicator = replicator; + this.filter = LogFilterManager.getLogFilter(filterName); + this.formatter = LogFormatterManager.getLogFormatter(formatterName); + + // this might take some time + remoteHost = socket.socket().getInetAddress().getHostName(); + + } + + /** + * Returns the remote hostname of this replicator connection. + * + * @return Returns the remote host name + */ + public String getRemoteHost() { + return remoteHost; + } + + /** + * Check if the message is wanted by this particular replicator + * connection. The reason we provide this method is because we + * want to be able to determine of a message is wanted by any + * client before committing resources to creating a ByteBuffer to + * serialize it into. + * + * @param msg The log message offered + * + */ + public boolean isLoggable (LogMessage msg) { + if (filter == null) { + return true; + } + return filter.isLoggable(msg); + } + + /** + * Return the description of the currently active filter. + */ + public String description () { + if (filter == null) { + return "No filter defined"; + } + return filter.description(); + } + + + /** + * Enqueues a ByteBuffer containing the message destined + * for the client. + * + * @param buffer the ByteBuffer into which the log message is + * serialized. + */ + public synchronized void enqueue (ByteBuffer buffer) throws IOException { + if (writeBuffer == null) { + writeBuffer = buffer; + } else { + // if we've reached the max we bail out + if (writeBufferList.size() > maxQueueLength) { + queueFullCount++; + if (! droppingMode) { + droppingMode = true; + String message = "client at " + remoteHost + " can't keep up, dropping messages"; + if (queueFullCount > maxRetriesBeforeWarning) { + log.log(LogLevel.WARNING, message); + queueFullCount = 0; + } else { + log.log(LogLevel.DEBUG, message); + } + } + numDropped++; + return; + } + writeBufferList.addLast(buffer); + listener.modifyInterestOps(this, SelectionKey.OP_WRITE, true); + droppingMode = false; + numQueued++; + } + numHandled++; + write(); + } + + + public void read () throws IOException { + if (! readBuffer.hasRemaining()) { + log.warning("Log message too long. Message exceeds " + + readBuffer.capacity() + + " bytes. Connection dropped."); + close(); + return; + } + + + int ret = socket.read(readBuffer); + if (ret == -1) { + close(); + return; + } + + if (ret == 0) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.fine("zero byte read occurred"); + } + } + + readBuffer.flip(); + + String s; + while ((s = ReadLine.readLine(readBuffer)) != null) { + onCommand(s); + } + + } + + public synchronized void write () throws IOException { + if (! socket.isOpen()) { + // throw new IllegalStateException("SocketChannel not open in write()"); + close(); + } + + int bytesWritten; + do { + // if writeBufferList is not set we need to fetch the next buffer + if (writeBuffer == null) { + + // if the list is empty, signal the selector we do not need + // to do any writing for a while yet and bail + if (writeBufferList.isEmpty()) { + listener.modifyInterestOpsBatch(this, + SelectionKey.OP_WRITE, + false); + return; + } + writeBuffer = writeBufferList.removeFirst(); + } + + + // invariants: we have a writeBuffer + + // when the client drops off we actually need + // to handle that here and close the connection + // XXX: I am not sure why this works and the + // close method call on IOException in + // Listener doesn't. this should be investigated! + // + try { + bytesWritten = socket.write(writeBuffer); + } + catch (IOException e) { + close(); + return; + } + totalBytesWritten += bytesWritten; + + // buffer drained so we forget it and see what happens when we + // go around. if indeed we go around + if ((writeBuffer != null) && (!writeBuffer.hasRemaining())) { + writeBuffer = null; + } + } while (bytesWritten > 0); + } + + public synchronized void close () throws IOException { + replicator.deRegisterConnection(this); + socket.close(); + writeBuffer = null; + writeBufferList.clear(); + log.log(LogLevel.DEBUG, "closing connection to " + remoteHost); + } + + public int selectOps () { + return SelectionKey.OP_READ; + } + + public SocketChannel socketChannel () { + return socket; + } + + public void connect () { + } + + + // ======================================================== + // ==== command processing + // ======================================================== + + void onCommand (String s) { + log.log(LogLevel.DEBUG, "COMMAND: '" + s + "' from " + remoteHost); + StringTokenizer st = new StringTokenizer(s.toLowerCase()); + while (st.hasMoreTokens()) { + String tok = st.nextToken(); + if ("ping".equals(tok)) { + if (st.hasMoreTokens()) { + print("# 202 pong " + st.nextToken() + "\n"); + } else { + print("# 202 pong\n"); + } + return; + } + + if ("use".equals(tok)) { + if (st.hasMoreTokens()) { + onUse(st.nextToken()); + } + return; + } + + if ("formatter".equals(tok)) { + if (st.hasMoreTokens()) { + onFormatter(st.nextToken()); + } + return; + } + + if ("quit".equals(tok)) { + print("# 201 bye\n"); + try { close(); } catch (IOException e) {e.printStackTrace();} + return; + } + + if ("list".equals(tok)) { + onList(); + return; + } + + if ("listformatters".equals(tok)) { + onListFormatters(); + return; + } + + if ("stats".equals(tok)) { + onStats(); + return; + } + } + } + + void onFormatter (String formatterName) { + LogFormatter newFormatter = LogFormatterManager.getLogFormatter(formatterName); + if (newFormatter == null) { + print("# 405 formatter not found '" + formatterName + "'\n"); + return; + } + formatter = newFormatter; + this.formatterName = formatterName; + print("# 202 using '" + formatter + "'\n"); + } + + void onUse (String filterName) { + LogFilter newFilter = LogFilterManager.getLogFilter(filterName); + if (newFilter == null) { + print("# 404 filter not found '" + filterName + "'\n"); + return; + } + filter = newFilter; + this.filterName = filterName; + print("# 200 using '" + filter + "'\n"); + } + + + + void onList () { + print("# 203 filter list\n"); + String filterNames[] = LogFilterManager.getFilterNames(); + for (int i = 0; i < filterNames.length; i++) { + LogFilter f = LogFilterManager.getLogFilter(filterNames[i]); + print("# 204 " + filterNames[i] + " - " + f.description() + "\n"); + } + print("# 205 end filter list\n"); + } + + void onListFormatters () { + print("# 206 formatter list\n"); + String formatterNames[] = LogFormatterManager.getFormatterNames(); + for (int i = 0; i < formatterNames.length; i++) { + LogFormatter fmt = LogFormatterManager.getLogFormatter(formatterNames[i]); + print("# 207 " + formatterNames[i] + " - " + fmt.description() + "\n"); + } + print("# 208 end formatter list\n"); + } + + private void print (String s) { + try { + enqueue(IOUtils.utf8ByteBuffer(s)); + } + catch (IOException e) { + log.log(LogLevel.WARNING, "error printing", e); + try { + close(); + } + catch (IOException e2) { + // ignore + } + } + } + + void onStats () { + + print(new StringBuilder(80) + .append("# 206 stats start (this connection)\n") + .append("# 207 ").append(numHandled).append(" handled\n") + .append("# 208 ").append(numDropped).append(" dropped\n") + .append("# 209 ").append(numQueued) + .append(" handled and queued\n") + .append("# 210 ").append(totalBytesWritten) + .append(" total bytes written\n") + .append("# 211 stats end\n") + .toString() + ); + } + + + public int getNumHandled () { + return numHandled; + } + + public int getNumQueued () { + return numQueued; + } + + public int getNumDropped () { + return numDropped; + } + + public long getTotalBytesWritten () { + return totalBytesWritten; + } + + public String getLogFilterName () { + return filterName; + } + + void setFilter(LogFilter filter) { + this.filter = filter; + } + +} + diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java new file mode 100644 index 00000000000..03bc66c8262 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.replicator; + +import java.io.IOException; +import java.util.logging.Logger; + +import com.yahoo.log.LogLevel; +import com.yahoo.logserver.Server; +import com.yahoo.plugin.Config; +import com.yahoo.plugin.Plugin; + +public class ReplicatorPlugin implements Plugin +{ + private static final String DEFAULT_PORT = "19083"; + private static final Logger log = + Logger.getLogger(ReplicatorPlugin.class.getName()); + private Replicator replicator; + private final Server server = Server.getInstance(); + + public String getPluginName () { + return "replicator"; + } + + /** + * Initialize the replicator plugin + */ + public void initPlugin (Config config) { + + if (replicator != null) { + throw new IllegalStateException( + "plugin already initialized: " + getPluginName()); + } + int listenPort = config.getInt("port", DEFAULT_PORT); + String threadName = config.get("thread", getPluginName()); + try { + replicator = new Replicator(listenPort); + } + catch (IOException e) { + log.log(LogLevel.WARNING, "init failed: " + e); + return; + } + server.registerLogHandler(replicator, threadName); + } + + /** + * Shut down the replicator plugin. + */ + public void shutdownPlugin () { + + if (replicator == null) { + throw new IllegalStateException( + "plugin not initialized: " + getPluginName()); + } + server.unregisterLogHandler(replicator); + replicator = null; + } + +} diff --git a/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java b/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java new file mode 100644 index 00000000000..1a48c5fb0f4 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java @@ -0,0 +1,272 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.net; + +import com.yahoo.log.LogLevel; +import com.yahoo.logserver.LogDispatcher; +import com.yahoo.log.LogMessage; +import com.yahoo.log.InvalidLogFormatException; + +import com.yahoo.io.Connection; +import com.yahoo.io.Listener; +import com.yahoo.io.ReadLine; + +import com.yahoo.logserver.net.control.Levels; + +import java.io.IOException; +import java.util.logging.Logger; +import java.util.logging.Level; +import java.util.LinkedList; +import java.util.Set; +import java.util.HashSet; + +import java.nio.charset.Charset; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.nio.channels.SelectionKey; + +/** + * TODO + *

    + *
  • send invalid log messages to somewhere so they can be + * analyzed and errors can be corrected. + *
+ * + * @author Bjorn Borud + */ + +public class LogConnection implements Connection { + private static final Logger log = Logger.getLogger(LogConnection.class.getName()); + + public static final int READBUFFER_SIZE = (32 * 1024); + + static private final Charset charset = Charset.forName("utf-8"); + + // the set of active connections + private static final Set activeConnections = new HashSet<>(); + + private final SocketChannel socket; + private final Listener listener; + private final LogDispatcher dispatcher; + + private final ByteBuffer readBuffer = ByteBuffer.allocateDirect(READBUFFER_SIZE); + + private final LinkedList writeBufferList = new LinkedList<>(); + private ByteBuffer writeBuffer; + + // counters + private long totalBytesRead = 0; + private long totalBytesWritten = 0; + + // default log levels for logd + final Levels defaultLevels; + + public LogConnection (SocketChannel socket, + Listener listener, + LogDispatcher dispatcher, + Levels defaultLevels) { + this.socket = socket; + this.listener = listener; + this.dispatcher = dispatcher; + this.defaultLevels = defaultLevels; + + addToActiveSet(this); + + // send the "setdefaultstate" command to logd. no better + // place to put it for now. + sendDefaultState(); + } + + /** + * Send the default state to the + */ + public void sendDefaultState () { + if (defaultLevels == null) { + return; + } + + try { + enqueue(charset.encode("setdefaultstate " + + defaultLevels.toString() + + "\n")); + enqueue(charset.encode("setallstates " + + defaultLevels.toString() + + "\n")); + } + catch (IOException e) { + log.log(LogLevel.WARNING, "Unable to send default state", e); + } + } + + /** + * Return a shallow copy of the set of active connections. + * + */ + public static Set getActiveConnections () { + synchronized(activeConnections) { + return new HashSet<>(activeConnections); + } + } + + /** + * @return Return total number of bytes written to connection + */ + public long getTotalBytesWritten () { + return totalBytesWritten; + } + + /** + * @return Return total number of bytes read from connection + */ + public long getTotalBytesRead () { + return totalBytesRead; + } + + /** + * Internal method for adding connection to the set + * of active connections. + * + * @param connection The connection to be added + */ + private static void addToActiveSet (LogConnection connection) { + synchronized(activeConnections) { + activeConnections.add(connection); + } + } + + /** + * Internal method to remove connection from the set of + * active connections. + * + * @param connection The connection to remove + * @throws IllegalStateException if the connection does not + * exist in the set + * + */ + private static void removeFromActiveSet (LogConnection connection) { + synchronized(activeConnections) { + activeConnections.remove(connection); + } + } + + /** + * + */ + public synchronized void enqueue (ByteBuffer buffer) throws IOException { + if (writeBuffer == null) { + writeBuffer = buffer; + } else { + writeBufferList.addLast(buffer); + listener.modifyInterestOps(this, SelectionKey.OP_WRITE, true); + } + write(); + } + + public void connect () throws IOException { + throw new RuntimeException("connect() is not supposed to be called"); + } + + public synchronized void write () throws IOException { + int bytesWritten; + do { + // if writeBufferList is not set we need to fetch the next buffer + if (writeBuffer == null) { + + // if the list is empty, signal the selector we do not need + // to do any writing for a while yet and bail + if (writeBufferList.isEmpty()) { + listener.modifyInterestOpsBatch(this, + SelectionKey.OP_WRITE, + false); + return; + } + writeBuffer = writeBufferList.removeFirst(); + } + + // invariants: we have a writeBuffer + + bytesWritten = socket.write(writeBuffer); + totalBytesWritten += bytesWritten; + + // buffer drained so we forget it and see what happens when we + // go around. if indeed we go around + if ((writeBuffer != null) && (!writeBuffer.hasRemaining())) { + writeBuffer = null; + } + } while (bytesWritten > 0); + } + + public void read() throws IOException { + if (! readBuffer.hasRemaining()) { + + try { + readBuffer.putChar(readBuffer.capacity() - 2, '\n'); + readBuffer.flip(); + String s = ReadLine.readLine(readBuffer); + if (s == null) { + return; + } + int count = 200; + log.log(LogLevel.WARNING, "Log message too long. Message from " + + socket.socket().getInetAddress() + " exceeds " + + readBuffer.capacity() + + ". Skipping buffer (might be part of same long message). Printing first " + count + " characters of line: " + + s.substring(0, count)); + + LogMessage msg = LogMessage.parseNativeFormat(s); + dispatcher.handle(msg); + } + catch (InvalidLogFormatException e) { + log.log(LogLevel.DEBUG, "Invalid log message", e); + } + finally { + readBuffer.clear(); + } + return; + } + + int ret = socket.read(readBuffer); + if (ret == -1) { + close(); + return; + } + + if (ret == 0) { + if (log.isLoggable(Level.FINE)) { + log.log(LogLevel.DEBUG, "zero byte read occurred"); + } + } + + // update global counter + totalBytesRead += ret; + + readBuffer.flip(); + + String s; + while ((s = ReadLine.readLine(readBuffer)) != null) { + try { + LogMessage msg = LogMessage.parseNativeFormat(s); + dispatcher.handle(msg); + } + catch (InvalidLogFormatException e) { + log.log(LogLevel.DEBUG, "Invalid log message", e); + } + } + } + + public void close() throws IOException { + if (log.isLoggable(Level.FINE)) { + log.log(LogLevel.INFO, this + ": closing"); + } + socket.close(); + removeFromActiveSet(this); + } + + public int selectOps() { + return SelectionKey.OP_READ; + } + + public SocketChannel socketChannel() { + return socket; + } + +} diff --git a/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java b/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java new file mode 100644 index 00000000000..5d97420f30a --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.net; + +import com.yahoo.logserver.LogDispatcher; +import com.yahoo.io.Connection; +import com.yahoo.io.Listener; +import com.yahoo.io.ConnectionFactory; + +import com.yahoo.logserver.net.control.Levels; + +import java.util.logging.Level; +import java.util.logging.Logger; +import java.nio.channels.SocketChannel; + +/** + * @author Bjorn Borud + */ + +public class LogConnectionFactory implements ConnectionFactory +{ + private static final Logger log + = Logger.getLogger(LogConnectionFactory.class.getName()); + + final LogDispatcher dispatcher; + final Levels defaultLogLevels; + + public LogConnectionFactory (LogDispatcher dispatcher) { + this.dispatcher = dispatcher; + defaultLogLevels = Levels.parse(System.getProperty("logserver.default.loglevels", "")); + } + + public Connection newConnection (SocketChannel socket, Listener listener) { + if (log.isLoggable(Level.FINE)) { + log.fine("New connection: " + socket); + } + return new LogConnection(socket, + listener, + dispatcher, + (Levels)defaultLogLevels.clone()); + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java b/logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java new file mode 100644 index 00000000000..c9b749f4417 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.net.control; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * This class is used to represent the state of each log level + * in a set of states. + * + * @author Bjorn Borud + */ +public class Levels implements Cloneable { + private final Map levelsMap = new LinkedHashMap(10); + + /** + * The constructor initializes the Levels object to its default + * state. + */ + public Levels () { + levelsMap.put("event", State.FORWARD); + levelsMap.put("fatal", State.FORWARD); + levelsMap.put("error", State.FORWARD); + levelsMap.put("warning", State.FORWARD); + levelsMap.put("info", State.FORWARD); + levelsMap.put("config", State.FORWARD); + levelsMap.put("debug", State.NOFORWARD); + levelsMap.put("spam", State.NOFORWARD); + } + + /** + * Parse a levels representation and return a Levels object + * representing the levels. + * + * @param levels A string representation of the levels + * @return new instance of Levels, possibly having no + * real values if none could be found in the + * levels parameter. + * + */ + public static Levels parse (String levels) { + return (new Levels()).updateLevels(levels); + } + + + /** + * Update the levels given a string representation of the state; + * the levels mentioned here will be updated, the ones omitted + * will retain their state as before the function call. + * + * @param levels string representation of levels + * + */ + public Levels updateLevels (String levels) { + String[] parts = levels.split(","); + if (parts.length < 1) { + return this; + } + + for (int i = 0; i < parts.length; i++) { + String pair = parts[i]; + int offset = pair.indexOf('='); + if (offset != -1) { + String name = pair.substring(0,offset).trim().toLowerCase(); + String value = pair.substring(offset+1).trim().toLowerCase(); + setLevelState(name, State.parse(value)); + } + } + return this; + } + + + /** + * Set the state of a given level. + * + * @param level name of the level + * @param state the state + * @return returns reference to this for chaning + */ + public Levels setLevelState(String level, State state) { + levelsMap.put(level, state); + return this; + } + + /** + * Get the state of a given level. + * + */ + public State getLevelState (String level) { + State s = levelsMap.get(level); + if (s == null) { + return State.UNKNOWN; + } + return s; + } + + /** + * For equivalent configurations the toString method should + * emit equal strings. + * + */ + public String toString () { + StringBuilder sbuf = new StringBuilder(80); + boolean first = true; + for (Map.Entry me : levelsMap.entrySet()) { + // commas between + if (!first) { + sbuf.append(','); + } else { + first = false; + } + sbuf.append(me.getKey()) + .append('=') + .append(me.getValue()); + } + + return sbuf.toString(); + } + + public Object clone() { + // quick and dirty, but easily verifiable to be correct + return parse(this.toString()); + } + +} diff --git a/logserver/src/main/java/com/yahoo/logserver/net/control/State.java b/logserver/src/main/java/com/yahoo/logserver/net/control/State.java new file mode 100644 index 00000000000..5d2a4f0fbd4 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/net/control/State.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.net.control; + +import java.util.Map; +import java.util.HashMap; + +/** + * This value class represents the logging state of a component. + * the valid states are: + * + *
    + *
  • forward - store locally and send to log server + *
  • store - store locally only + *
  • noforward - do not send to logserver + *
  • off - do not generate the message in the program + *
+ * + * XXX This does not appear to be in use. + */ +public class State { + private static final Map nameToState = new HashMap(); + + public static final State FORWARD = new State("forward"); + public static final State NOFORWARD = new State("noforward"); + // public static final State STORE = new State("store"); + // public static final State OFF = new State("off"); + public static final State UNKNOWN = new State("unknown"); + + private String name; + + /** + * Typesafe enum. Only able to instantiate self. + * TODO: Rewrite to enum + */ + private State () {} + + /** + * Creates state with given name + */ + private State (String name) { + this.name = name; + synchronized (State.class) { + nameToState.put(name, this); + } + } + + public static State parse (String s) { + return nameToState.containsKey(s) ? nameToState.get(s) : UNKNOWN; + } + + public String toString () { + return name; + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java new file mode 100644 index 00000000000..623bf04aa36 --- /dev/null +++ b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.testutils; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; + +import com.yahoo.log.LogLevel; +import com.yahoo.log.event.MalformedEventException; +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogMessage; + +/** + * This utility is used to check that the log messages contained + * in a log file are correct. Any incorrectly formatted log + * message is output to stdout. + * + * @author Bjorn Borud + */ +public class VerifyLogfile { + + public static void main (String[] args) throws IOException { + int messages = 0; + int events = 0; + int invalidLogMessages = 0; + int invalidEvents = 0; + int numFiles = 0; + + if (args.length < 1) { + System.err.println("\nPlease provide name of log file(s)\n"); + } + + for (int i = 0; i < args.length; i++) { + BufferedReader br = new BufferedReader(new FileReader(args[i])); + numFiles++; + for (String line = br.readLine(); + line != null; + line = br.readLine()) + { + messages++; + LogMessage m; + try { + m = LogMessage.parseNativeFormat(line); + if (m.getLevel() == LogLevel.EVENT) { + events++; + m.getEvent(); + } + } catch (MalformedEventException e) { + System.out.println("EVENT\t" + line); + invalidEvents++; + } catch (InvalidLogFormatException e) { + System.out.println("MESSAGE\t" + line); + invalidLogMessages++; + } + } + br.close(); + } + + System.err.println("numFiles: " + numFiles); + System.err.println("messages: " + messages); + System.err.println("events: " + events); + System.err.println("invalidLogMessages: " + invalidLogMessages); + System.err.println("invalidEvents: " + invalidEvents); + } +} diff --git a/logserver/src/main/java/com/yahoo/plugin/Config.java b/logserver/src/main/java/com/yahoo/plugin/Config.java new file mode 100644 index 00000000000..a7ea2b44caf --- /dev/null +++ b/logserver/src/main/java/com/yahoo/plugin/Config.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.plugin; + +/** + * This interface specifies an API for configuring runtime-loadable + * server plugins. + * + * @author Stig Bakken + */ +public abstract class Config +{ + /** + * @return a config value for the specified key + */ + public abstract String get(String key, String defaultValue); + + /** + * @return a config value as an integer + */ + public int getInt(String key, String defaultValue) { + return Integer.parseInt(get(key, defaultValue)); + } +} diff --git a/logserver/src/main/java/com/yahoo/plugin/Plugin.java b/logserver/src/main/java/com/yahoo/plugin/Plugin.java new file mode 100644 index 00000000000..e53bc3a1faa --- /dev/null +++ b/logserver/src/main/java/com/yahoo/plugin/Plugin.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.plugin; + +/** + * This interface specifies an API for runtime-loadable server + * plugins. The interface is deliberately simple to allow it to be + * used in different servers. Typically, the initPlugin() method + * calls application-specific registration methods to connect the + * plugin to the hosting application. + * + * @author Stig Bakken + */ +public interface Plugin +{ + /** + * @return a unique and simple name for the plugin + */ + public String getPluginName(); + + /** + * Initialize the plugin. + * + * @param config Config object for this plugin + */ + public void initPlugin(Config config); + + /** + * Shut down the plugin. Must clean up all resources allocated by + * initPlugin() or any of the handler methods. + */ + public void shutdownPlugin(); + +} diff --git a/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java b/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java new file mode 100644 index 00000000000..763dc2d1f3e --- /dev/null +++ b/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.plugin; + +/** + * This class implements plugin config through system properties. + * Each plugin typically has its own system property prefix, such as + * "logserver.archiver.". A request for the config key "foo" will + * then return the contents of the "logserver.archiver.foo" system + * property. + * + * @author Stig Bakken + */ +public class SystemPropertyConfig extends Config +{ + private final String prefix; + + /** + * @param prefix Prefix string prepended to config keys + * as they are looked up as system properties. + */ + public SystemPropertyConfig(String prefix) { + this.prefix = prefix; + } + + /** + * @return a config value for the specified key + */ + public String get(String key, String defaultValue) { + return System.getProperty(prefix + key, defaultValue); + } + + public String toString () { + return "Prefix=" + prefix; + } +} diff --git a/logserver/src/test/files/event.txt.gz b/logserver/src/test/files/event.txt.gz new file mode 100644 index 00000000000..9422ca5fc90 Binary files /dev/null and b/logserver/src/test/files/event.txt.gz differ diff --git a/logserver/src/test/files/logEntries.txt b/logserver/src/test/files/logEntries.txt new file mode 100644 index 00000000000..776e3de882d --- /dev/null +++ b/logserver/src/test/files/logEntries.txt @@ -0,0 +1,3 @@ +1096639280.524133 malfunction 26851 - logtest info Starting up, called as ./log/logtest +1096639280.524164 malfunction 26851 - logtest info backslash: \\ +1096639280.524133 malfunction 26851 - logtest fatal Starting up, called as ./log/logtest diff --git a/logserver/src/test/files/value-events.txt b/logserver/src/test/files/value-events.txt new file mode 100644 index 00000000000..20143d9cd53 --- /dev/null +++ b/logserver/src/test/files/value-events.txt @@ -0,0 +1,982 @@ +1107248423.987624 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248423.987754 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248423.987809 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248439.235755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248439.235871 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248439.236632 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248439.236684 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248423.987624 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248423.987754 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248423.987809 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248439.235755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248439.235871 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248439.236632 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248439.236684 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248439.345800 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248439.345924 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248454.346566 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248454.346701 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248454.346755 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248484.708180 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248484.708309 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248484.708362 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248499.377732 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248499.377855 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248499.378618 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248499.378670 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248499.379345 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248499.379407 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248515.068667 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248515.068795 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248515.068848 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248545.429750 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248545.429879 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248545.429933 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248559.509728 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248559.509849 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248559.510318 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248559.510382 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248559.511135 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248559.511187 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248575.790604 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248575.790728 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248575.790780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248606.151589 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248606.151717 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248606.151769 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248619.601700 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248619.601824 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248619.602250 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248619.602314 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248619.603067 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248619.603119 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248636.512734 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248636.512861 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248636.512915 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248666.873517 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248666.873740 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248666.873796 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248679.703659 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248679.703779 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248679.704182 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248679.704244 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248679.704995 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248679.705048 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248697.234524 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248697.234653 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248697.234708 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248727.595687 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248727.595817 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248727.595870 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248739.755755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248739.755880 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248739.756771 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248739.756824 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248739.865626 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248739.865746 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248757.956408 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248757.956542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248757.956595 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248788.317656 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248788.317784 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248788.317838 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248799.887618 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248799.887747 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248799.888510 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248799.888563 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248799.888865 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248799.889013 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248818.678404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248818.678542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248818.678607 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248849.039894 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248849.040024 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248849.040079 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248859.959613 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248859.959735 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248859.960498 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248859.960550 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248860.069650 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248860.069875 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248879.400552 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248879.400678 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248879.400732 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248909.761338 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248909.761479 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248909.761537 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248920.061583 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248920.061705 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248920.062472 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248920.062525 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248920.171575 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248920.171695 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248940.122556 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248940.122697 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248940.122751 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248970.483540 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248970.483669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248970.483723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248980.223600 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248980.223719 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248980.224483 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248980.224536 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248980.225291 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248980.225386 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249000.844582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249000.844711 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249000.844765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249031.205764 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249031.205990 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249031.206047 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249040.295520 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249040.295643 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249040.296405 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249040.296457 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249040.297130 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249040.297192 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249061.566458 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249061.566586 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249061.566640 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249091.927910 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249091.928037 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249091.928091 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249100.327467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249100.327593 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249100.328494 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249100.328547 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249100.457718 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249100.457824 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249122.288543 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249122.288671 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249122.288724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249152.649685 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249152.649816 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249152.649869 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249160.509467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249160.509544 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249160.509615 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249160.509708 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249160.510802 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249160.510855 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249183.011765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249183.011895 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249183.011948 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249213.371426 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249213.371565 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249213.371622 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249220.551434 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249220.551563 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249220.552329 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249220.552381 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249220.661500 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249220.661627 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249243.732724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249243.732852 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249243.732906 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249274.093649 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249274.093780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249274.093833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249280.663427 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249280.663550 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249280.664313 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249280.664365 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249280.773405 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249280.773526 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249304.454704 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249304.454833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249304.454886 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249334.815726 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249334.815858 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249334.815912 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249340.825357 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249340.825488 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249340.826289 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249340.826342 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249340.827122 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249340.827187 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249365.177542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249365.177669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249365.177723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249395.541404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249395.541529 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249395.541582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249482.760126 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249482.760259 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249482.760989 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249482.761042 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249484.460156 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249484.460310 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249521.879344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249521.879842 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249521.879913 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249542.731983 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249542.732513 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249542.733591 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249542.733644 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249544.402200 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249544.402588 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249552.232878 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249552.233293 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249552.233356 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249582.593375 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249582.594022 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249582.594304 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249602.934015 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249602.935138 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249602.936242 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249602.936310 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249604.514022 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249604.514411 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249612.954767 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249612.954976 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249612.955162 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249643.315410 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249643.315971 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249643.316213 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249662.938231 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249662.938636 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249662.939500 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249662.939552 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249664.606105 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249664.606520 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249673.676574 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249673.677044 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249673.677282 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249704.037945 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249704.038370 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249704.038433 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249723.107946 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249723.108405 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249723.109464 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249723.109518 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249724.667973 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249724.668503 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249734.398557 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249734.399061 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249734.399424 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249764.760077 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249764.760499 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249764.760562 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249783.149953 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249783.150509 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249783.151530 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249783.151610 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249784.860115 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249784.860543 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249795.120517 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249795.121115 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249795.121470 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249825.481800 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249825.482123 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249825.482265 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249843.301904 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249843.303003 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249843.304400 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249843.304486 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249844.971993 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249844.972487 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249855.842709 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249855.843038 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249855.843176 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249886.203512 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249886.203987 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249886.204214 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249903.433833 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249903.434091 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249903.435239 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249903.435292 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249905.023913 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249905.024305 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249916.564719 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249916.565133 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249916.565277 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249946.925416 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249946.925938 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249946.926367 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249963.525789 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249963.526251 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249963.527498 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249963.527562 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249965.085940 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249965.086355 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249977.286716 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249977.286985 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249977.287134 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250007.647650 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250007.648010 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250007.648158 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250023.557752 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250023.558227 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250023.559250 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250023.559334 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250025.247819 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250025.248795 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250038.008626 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250038.008972 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250038.009117 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250068.369715 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250068.370110 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250068.370631 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250083.689727 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250083.690303 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250083.691188 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250083.691300 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250085.359965 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250085.360376 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250098.730564 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250098.732151 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250098.732322 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250129.092925 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250129.093340 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250129.093403 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250143.761756 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250143.762207 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250143.763278 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250143.763355 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250145.471889 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250145.472434 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250159.453836 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250159.454250 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250159.454313 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250189.813551 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250189.814359 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250189.814797 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250203.923674 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250203.924112 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250203.925109 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250203.925196 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250205.513914 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250205.514315 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250220.176334 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250220.176748 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250220.176811 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250250.535671 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250250.536188 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250250.536510 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250263.945671 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250263.945967 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250263.947065 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250263.947151 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250265.655706 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250265.656171 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250280.896680 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250280.897147 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250280.897565 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250311.257655 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250311.258113 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250311.258344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250324.067765 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250324.068159 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250324.068986 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250324.069111 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250325.727674 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250325.728124 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248423.987624 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248423.987754 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248423.987809 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248439.235755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248439.235871 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248439.236632 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248439.236684 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248439.345800 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248439.345924 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248454.346566 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248454.346701 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248454.346755 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248484.708180 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248484.708309 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248484.708362 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248499.377732 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248499.377855 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248499.378618 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248499.378670 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248499.379345 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248499.379407 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248515.068667 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248515.068795 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248515.068848 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248545.429750 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248545.429879 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248545.429933 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248559.509728 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248559.509849 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248559.510318 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248559.510382 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248559.511135 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248559.511187 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248575.790604 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248575.790728 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248575.790780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248606.151589 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248606.151717 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248606.151769 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248619.601700 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248619.601824 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248619.602250 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248619.602314 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248619.603067 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248619.603119 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248636.512734 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248636.512861 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248636.512915 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248666.873517 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248666.873740 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248666.873796 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248679.703659 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248679.703779 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248679.704182 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248679.704244 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248679.704995 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248679.705048 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248697.234524 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248697.234653 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248697.234708 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248727.595687 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248727.595817 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248727.595870 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248739.755755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248739.755880 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248739.756771 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248739.756824 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248739.865626 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248739.865746 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248757.956408 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248757.956542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248757.956595 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248788.317656 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248788.317784 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248788.317838 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248799.887618 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248799.887747 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248799.888510 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248799.888563 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248799.888865 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248799.889013 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248818.678404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248818.678542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248818.678607 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248849.039894 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248849.040024 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248849.040079 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248859.959613 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248859.959735 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248859.960498 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248859.960550 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248860.069650 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248860.069875 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248879.400552 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248879.400678 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248879.400732 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248909.761338 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248909.761479 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248909.761537 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248920.061583 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248920.061705 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248920.062472 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248920.062525 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248920.171575 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248920.171695 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107248940.122556 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248940.122697 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248940.122751 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248970.483540 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107248970.483669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107248970.483723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107248980.223600 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107248980.223719 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107248980.224483 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107248980.224536 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107248980.225291 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107248980.225386 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249000.844582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249000.844711 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249000.844765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249031.205764 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249031.205990 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249031.206047 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249040.295520 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249040.295643 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249040.296405 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249040.296457 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249040.297130 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249040.297192 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249061.566458 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249061.566586 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249061.566640 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249091.927910 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249091.928037 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249091.928091 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249100.327467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249100.327593 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249100.328494 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249100.328547 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249100.457718 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249100.457824 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249122.288543 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249122.288671 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249122.288724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249152.649685 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249152.649816 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249152.649869 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249160.509467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249160.509544 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249160.509615 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249160.509708 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249160.510802 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249160.510855 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249183.011765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249183.011895 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249183.011948 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249213.371426 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249213.371565 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249213.371622 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249220.551434 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249220.551563 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249220.552329 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249220.552381 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249220.661500 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249220.661627 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249243.732724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249243.732852 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249243.732906 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249274.093649 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249274.093780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249274.093833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249280.663427 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249280.663550 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249280.664313 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249280.664365 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249280.773405 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249280.773526 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249304.454704 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249304.454833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249304.454886 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249334.815726 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249334.815858 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249334.815912 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249340.825357 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249340.825488 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249340.826289 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249340.826342 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249340.827122 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249340.827187 example.yahoo.com 28552 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249365.177542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249365.177669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249365.177723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249395.541404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249395.541529 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249395.541582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249482.760126 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249482.760259 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249482.760989 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249482.761042 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249484.460156 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249484.460310 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249521.879344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249521.879842 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249521.879913 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249542.731983 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249542.732513 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249542.733591 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249542.733644 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249544.402200 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249544.402588 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249552.232878 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249552.233293 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249552.233356 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249582.593375 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249582.594022 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249582.594304 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249602.934015 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249602.935138 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249602.936242 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249602.936310 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249604.514022 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249604.514411 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249612.954767 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249612.954976 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249612.955162 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249643.315410 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249643.315971 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249643.316213 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249662.938231 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249662.938636 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249662.939500 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249662.939552 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249664.606105 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249664.606520 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249673.676574 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249673.677044 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249673.677282 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249704.037945 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249704.038370 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249704.038433 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249723.107946 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249723.108405 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249723.109464 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249723.109518 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249724.667973 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249724.668503 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249734.398557 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249734.399061 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249734.399424 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249764.760077 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249764.760499 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249764.760562 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249783.149953 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249783.150509 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249783.151530 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249783.151610 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249784.860115 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249784.860543 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249795.120517 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249795.121115 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249795.121470 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249825.481800 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249825.482123 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249825.482265 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249843.301904 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249843.303003 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249843.304400 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249843.304486 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249844.971993 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249844.972487 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249855.842709 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249855.843038 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249855.843176 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249886.203512 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249886.203987 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249886.204214 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249903.433833 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249903.434091 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249903.435239 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249903.435292 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249905.023913 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249905.024305 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249916.564719 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249916.565133 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249916.565277 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249946.925416 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249946.925938 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249946.926367 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107249963.525789 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107249963.526251 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107249963.527498 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107249963.527562 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107249965.085940 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107249965.086355 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107249977.286716 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107249977.286985 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107249977.287134 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250007.647650 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250007.648010 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250007.648158 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250023.557752 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250023.558227 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250023.559250 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250023.559334 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250025.247819 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250025.248795 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250038.008626 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250038.008972 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250038.009117 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250068.369715 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250068.370110 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250068.370631 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250083.689727 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250083.690303 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250083.691188 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250083.691300 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250085.359965 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250085.360376 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250098.730564 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250098.732151 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250098.732322 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250129.092925 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250129.093340 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250129.093403 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250143.761756 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250143.762207 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250143.763278 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250143.763355 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250145.471889 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250145.472434 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250159.453836 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250159.454250 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250159.454313 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250189.813551 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250189.814359 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250189.814797 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250203.923674 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250203.924112 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250203.925109 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250203.925196 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250205.513914 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250205.514315 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250220.176334 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250220.176748 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250220.176811 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250250.535671 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250250.536188 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250250.536510 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250263.945671 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250263.945967 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250263.947065 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250263.947151 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250265.655706 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250265.656171 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250280.896680 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250280.897147 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250280.897565 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250311.257655 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250311.258113 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250311.258344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250324.067765 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250324.068159 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250324.068986 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250324.069111 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250325.727674 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250325.728124 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250341.618469 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250341.619001 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250341.619367 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250371.979550 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250371.980003 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250371.980254 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250384.209615 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250384.210153 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250384.211269 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250384.211322 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250385.789685 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250385.790201 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250402.340723 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250402.341145 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250402.341208 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250432.701722 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250432.702052 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250432.702193 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250444.241654 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250444.242282 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250444.243410 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250444.243524 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250445.921705 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250445.922178 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250463.062779 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250463.063193 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250463.063257 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250493.423518 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250493.423869 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250493.424011 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250504.433587 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250504.434091 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250504.435178 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250504.435231 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250506.013628 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250506.013991 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250523.791832 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250523.791966 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250523.792019 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250554.145291 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250554.145429 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250554.145494 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250564.525521 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250564.525632 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250564.526387 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250564.526439 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250566.085563 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250566.085687 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250584.506801 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250584.506957 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250584.507011 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250614.868677 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250614.868807 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250614.868861 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250624.567617 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250624.567734 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250624.568485 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250624.568536 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250626.257564 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250626.257686 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250645.229659 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250645.229812 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250645.229866 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250675.589502 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250675.589644 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250675.589708 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250684.729465 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250684.729580 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250684.730461 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250684.730512 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250686.299574 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250686.299692 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250705.950505 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250705.950692 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250705.950769 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250736.311593 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250736.311734 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250736.311798 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250744.761494 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250744.761626 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250744.762381 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250744.762433 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250746.441505 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250746.441626 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250766.672522 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250766.672708 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250766.672763 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250797.034486 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250797.034617 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250797.034670 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250804.863485 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250804.863613 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250804.864360 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250804.864411 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250806.573649 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250806.573769 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250827.395696 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250827.395823 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250827.395876 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250857.756404 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250857.756533 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250857.756585 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250865.035459 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250865.035581 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250865.036337 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250865.036388 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250866.595476 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250866.595599 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250888.116534 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250888.116674 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250888.116737 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250918.479148 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250918.479277 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250918.479332 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250925.137420 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250925.137544 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250925.138351 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250925.138403 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250926.717467 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250926.717575 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107250948.838645 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250948.838779 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250948.838836 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250979.199468 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107250979.199645 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107250979.199705 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107250985.130997 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107250985.131117 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107250985.131975 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107250985.132027 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107250986.809443 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107250986.809566 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107251009.560525 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251009.560664 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251009.560727 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251039.921739 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251039.921872 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251039.921926 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251045.311481 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107251045.311598 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107251045.312351 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107251045.312403 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107251046.871495 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107251046.871619 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107251070.282709 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251070.282849 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251070.282918 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251100.643735 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251100.643875 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251100.643928 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251105.403383 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107251105.403514 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107251105.404268 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107251105.404320 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107251106.983407 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107251106.983527 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107251131.004511 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251131.004656 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251131.004753 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251161.365529 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251161.365664 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251161.365721 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251165.455342 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107251165.455454 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107251165.456208 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107251165.456260 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107251167.135419 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107251167.135523 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107251191.726752 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251191.726890 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251191.726957 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251222.087961 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251222.088092 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251222.088146 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251225.587446 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107251225.587559 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107251225.588311 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107251225.588363 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107251227.167723 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107251227.167846 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107251252.448823 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251252.448959 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251252.449014 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251282.809626 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251282.809768 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251282.809831 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251285.629323 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107251285.629447 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107251285.630309 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107251285.630361 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107251287.319473 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107251287.319591 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107251313.170437 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251313.170575 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251313.170631 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251343.531690 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251343.531826 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251343.531880 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251345.811348 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107251345.811464 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107251345.812220 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107251345.812272 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107251347.391374 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107251347.391481 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107251373.892751 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251373.892890 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251373.892954 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251404.253471 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251404.253618 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251404.253675 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251405.923299 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107251405.923418 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107251405.924170 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107251405.924221 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107251407.513274 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107251407.513397 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107251434.614618 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251434.614759 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251434.614823 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251464.975470 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251464.975607 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251464.975662 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251466.005315 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107251466.005448 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107251466.006208 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107251466.006260 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107251467.565334 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107251467.565455 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107251495.336518 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251495.336658 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251495.336721 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251525.697793 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251525.697922 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251525.697975 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 +1107251526.047229 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 +1107251526.047353 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 +1107251526.048109 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 +1107251526.048161 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 +1107251527.737300 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="queued" value=0 +1107251527.737423 example.yahoo.com 59773 tld fdispatch.queryperf event value/1 name="active" value=0 +1107251556.058595 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 +1107251556.058737 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 +1107251556.058848 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 diff --git a/logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java b/logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java new file mode 100644 index 00000000000..d74504b5ed0 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.yahoo.logserver.handlers.LogHandler; +import com.yahoo.logserver.test.LogDispatcherTestCase; + +public class FlusherTestCase { + + @Test + public void testFlusher() throws InterruptedException { + Flusher flusher = new Flusher(); + LogDispatcherTestCase.MockHandler handler = new LogDispatcherTestCase.MockHandler(); + flusher.register(handler); + flusher.start(); + Thread.sleep(5000); + flusher.interrupt(); + flusher.join(); + assertTrue(handler.flushCalled>0); + } + +} diff --git a/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java new file mode 100644 index 00000000000..1a6cd304d79 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver; + +import com.yahoo.log.LogSetup; +import com.yahoo.logserver.Server; +import com.yahoo.logserver.handlers.LogHandler; +import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin; +import com.yahoo.logserver.test.LogDispatcherTestCase; + +import java.io.IOException; + +import org.junit.*; +import static org.junit.Assert.*; + +/** + * Unit tests for the Server class. + * + * @author Bjorn Borud + */ +public class ServerTestCase { + + @Test + public void testStartupAndRegHandlers() throws IOException, InterruptedException { + Server.help(); + Server server = Server.getInstance(); + server.initialize(18322); + LogSetup.clearHandlers(); + Thread serverThread = new Thread(server); + serverThread.start(); + assertTrue(serverThread.isAlive()); + LogHandler handler = new LogDispatcherTestCase.MockHandler(); + server.registerLogHandler(handler , "foo"); + assertEquals(Server.threadNameForHandler().get(handler), "foo"); + server.unregisterLogHandler(handler); + assertEquals(Server.threadNameForHandler().get(handler), null); + serverThread.interrupt(); + try { + serverThread.join(); + assertTrue(true); + } + catch (InterruptedException e) { + fail(); + } + } + + @Test + public void testPluginLoaderClassLoading() { + AbstractPluginLoader loader = new BuiltinPluginLoader(); + System.setProperty("logserver.logmetrics.enable", "false"); + loader.loadFromClass(LogMetricsPlugin.class); + System.setProperty("logserver.logmetrics.enable", "true"); + loader.loadFromClass(LogMetricsPlugin.class); // Hm, no way to verify it was loaded + } + +} diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java new file mode 100644 index 00000000000..439c8e3978f --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.filter.test; + +import com.yahoo.logserver.filter.LogFilter; +import com.yahoo.logserver.filter.LogFilterManager; +import com.yahoo.logserver.filter.LevelFilter; +import com.yahoo.logserver.filter.MetricsFilter; +import com.yahoo.logserver.filter.NoMetricsFilter; +import com.yahoo.logserver.filter.NullFilter; +import com.yahoo.logserver.filter.MuteFilter; + +import org.junit.*; +import static org.junit.Assert.*; + +public class LogFilterManagerTestCase { + + @Test + public void testSystemFilters () { + LogFilter f; + + f = LogFilterManager.getLogFilter("system.allevents"); + assertNotNull(f); + assertTrue(f instanceof LevelFilter); + + f = LogFilterManager.getLogFilter("system.metricsevents"); + assertNotNull(f); + assertTrue(f instanceof MetricsFilter); + + f = LogFilterManager.getLogFilter("system.nometricsevents"); + assertNotNull(f); + assertTrue(f instanceof NoMetricsFilter); + + + f = LogFilterManager.getLogFilter("system.all"); + assertNotNull(f); + assertTrue(f instanceof NullFilter); + + f = LogFilterManager.getLogFilter("system.mute"); + assertNotNull(f); + assertTrue(f instanceof MuteFilter); + assertTrue(f == MuteFilter.getInstance()); + } +} diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java new file mode 100644 index 00000000000..fa22daac0d8 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.filter.test; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; + +import com.yahoo.log.event.Event; +import com.yahoo.log.event.MalformedEventException; +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.filter.MetricsFilter; + +import org.junit.*; +import static org.junit.Assert.*; + +public class MetricsFilterTestCase { + + @Test + public void testValueEvents () + throws InvalidLogFormatException, IOException { + MetricsFilter filter = new MetricsFilter(); + String filename = "src/test/files/value-events.txt"; + BufferedReader br = new BufferedReader(new FileReader(filename)); + for (String line = br.readLine(); line != null; line = br.readLine()) + { + LogMessage m = LogMessage.parseNativeFormat(line); + assertNotNull(m); + try { + Event event = m.getEvent(); + assertNotNull(event); + } + catch (MalformedEventException e) { + fail(); + } + + if (filter.isLoggable(m)) { + assertTrue(true); + } else { + fail(); + } + } + + } +} diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java new file mode 100644 index 00000000000..5d6be5c16c7 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.filter.test; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; + +import com.yahoo.log.event.Event; +import com.yahoo.log.event.MalformedEventException; +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.filter.MetricsFilter; +import com.yahoo.logserver.filter.NoMetricsFilter; + +import org.junit.*; +import static org.junit.Assert.*; + +/** + * Ensure that the NoMetricsFilter does the opposite of MetricsFilter + */ +public class NoMetricsFilterTestCase { + + @Test + public void testValueEvents () throws InvalidLogFormatException, IOException { + NoMetricsFilter filter = new NoMetricsFilter(); + MetricsFilter metricsFilter = new MetricsFilter(); + + String filename = "src/test/files/value-events.txt"; + BufferedReader br = new BufferedReader(new FileReader(filename)); + for (String line = br.readLine(); line != null; line = br.readLine()) { + LogMessage m = LogMessage.parseNativeFormat(line); + assertNotNull(m); + + try { + Event event = m.getEvent(); + assertNotNull(event); + } + catch (MalformedEventException e) { + fail(); + } + + if (filter.isLoggable(m)) { + fail(); + } else { + assertTrue(true); + } + + + if (metricsFilter.isLoggable(m)) { + assertTrue(true); + } else { + fail(); + } + } + } +} diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java new file mode 100644 index 00000000000..288300f9251 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * $Id$ + * + */ +package com.yahoo.logserver.formatter.test; + +import com.yahoo.logserver.formatter.LogFormatter; +import com.yahoo.logserver.formatter.LogFormatterManager; +import com.yahoo.logserver.formatter.NullFormatter; +import com.yahoo.logserver.formatter.TextFormatter; + +import org.junit.*; +import static org.junit.Assert.*; + +/** + * Test the LogFormatterManager + * + * @author Bjorn Borud + */ +public class LogFormatterManagerTestCase { + + /** + * Ensure the system formatters are present + */ + @Test + public void testSystemFormatters() { + LogFormatter lf = LogFormatterManager.getLogFormatter("system.textformatter"); + assertNotNull(lf); + assertEquals(TextFormatter.class, lf.getClass()); + + lf = LogFormatterManager.getLogFormatter("system.nullformatter"); + assertNotNull(lf); + assertEquals(NullFormatter.class, lf.getClass()); + } +} diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java new file mode 100644 index 00000000000..b0fc7ea09c0 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * $Id$ + * + */ +package com.yahoo.logserver.formatter.test; + +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.formatter.NullFormatter; +import com.yahoo.logserver.test.MockLogEntries; + +import org.junit.*; +import static org.junit.Assert.*; + +/** + * Test the NullFormatter + * + * @author Bjorn Borud + */ +public class NullFormatterTestCase { + + @Test + public void testNullFormatter () { + NullFormatter nf = new NullFormatter(); + LogMessage[] ms = MockLogEntries.getMessages(); + for (LogMessage m : ms) { + assertEquals(m.toString(), nf.format(m)); + } + } +} diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java new file mode 100644 index 00000000000..2e7e2d38b57 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * $Id$ + * + */ +package com.yahoo.logserver.formatter.test; + +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.formatter.TextFormatter; +import com.yahoo.logserver.test.MockLogEntries; + +import org.junit.*; +import static org.junit.Assert.*; + +/** + * Test the TextFormatter + * + * @author Bjorn Borud + */ +public class TextFormatterTestCase { + + /** + * Just simple test to make sure it doesn't die on us + */ + @Test + public void testTextFormatter () { + TextFormatter tf = new TextFormatter(); + LogMessage[] ms = MockLogEntries.getMessages(); + for (int i = 0; i < ms.length; i++) { + System.out.println(tf.format(ms[i])); + } + } + + /** + * Test that a specific log message is formatted correctly + */ + @Test + public void testSpecificMessage () throws InvalidLogFormatException { + String l = "1115200798.195568\texample.yahoo.com\t65819\ttopleveldispatch\tfdispatch.queryperf\tevent\tvalue/1 name=\"query_eval_time_avg_s\" value=0.0229635972697721825"; + String result = "2005-05-04 09:59:58 example.yahoo.com 65819 topleveldispatch fdispatch.queryperf EVENT value/1 name=\"query_eval_time_avg_s\" value=0.0229635972697721825\n"; + LogMessage m = LogMessage.parseNativeFormat(l); + TextFormatter tf = new TextFormatter(); + assertEquals(result, tf.format(m)); + } +} diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java new file mode 100644 index 00000000000..16c1c780bb1 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers; + +import static org.junit.Assert.*; + +import java.util.ArrayList; + +import org.junit.Test; + +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.test.LogDispatcherTestCase; + +public class HandlerThreadTestCase { + + @Test + public void testHandlerThread() throws InterruptedException, InvalidLogFormatException { + HandlerThread thread = new HandlerThread("myThread"); + LogDispatcherTestCase.MockHandler h1 = new LogDispatcherTestCase.MockHandler(); + LogDispatcherTestCase.MockHandler h2 = new LogDispatcherTestCase.MockHandler(); + LogDispatcherTestCase.MockHandler h3 = new LogDispatcherTestCase.MockHandler(); + thread.registerHandler(h1); + thread.registerHandler(h2); + thread.registerHandler(h3); + thread.start(); + String msgstr1 = "1098709001\t" + + "nalle.puh.com\t" + + "23234\t" + + "serviceName\t" + + "tst\t" + + "info\t" + + "this is a test"; + String msgstr2 = "1098709002\t" + + "nalle.puh.com\t" + + "23234\t" + + "serviceName1\t" + + "tst\t" + + "info\t" + + "this is a test too"; + String msgstr3 = "1098709003\t" + + "nalle.puh.com\t" + + "23234\t" + + "serviceName3\t" + + "tst\t" + + "info\t" + + "this is a test also"; + final LogMessage msg1 = LogMessage.parseNativeFormat(msgstr1); + final LogMessage msg2 = LogMessage.parseNativeFormat(msgstr2); + final LogMessage msg3 = LogMessage.parseNativeFormat(msgstr3); + thread.handle(msg1); + while ((h1.messages.size() < 1) || (h2.messages.size() < 1) || (h3.messages.size() < 1)) { + thread.sleep(10); + } + assertEquals(h1.messages.size(), 1); + assertEquals(h2.messages.size(), 1); + assertEquals(h3.messages.size(), 1); + thread.handle(new ArrayList() {{add(msg1); add(msg2); add(msg3); }}); + while ((h1.messages.size() < 4) || (h2.messages.size() < 4) || (h3.messages.size() < 4)) { + thread.sleep(10); + } + assertEquals(h1.messages.size(), 4); + assertEquals(h2.messages.size(), 4); + assertEquals(h3.messages.size(), 4); + assertTrue(thread.getQueue().isEmpty()); + thread.unregisterHandler(h3); + assertEquals(thread.getHandlers().length, 2); + thread.flush(); + thread.close(); + thread.interrupt(); + thread.join(); + } + + @Test + public void testAbortThread() throws InvalidLogFormatException, InterruptedException { + HandlerThread thread = new HandlerThread("myThread"); + LogDispatcherTestCase.MockHandler h1 = new LogDispatcherTestCase.MockHandler(); + thread.registerHandler(h1); + thread.start(); + String msgstr1 = "1098709001\t" + + "nalle.puh.com\t" + + "23234\t" + + "serviceName\t" + + "tst\t" + + "info\t" + + "this is a test"; + final LogMessage msg1 = LogMessage.parseNativeFormat(msgstr1); + thread.handle(msg1); + while (h1.messages.size() < 1) { + thread.sleep(10); + } + assertEquals(h1.messages.size(), 1); + thread.interrupt(); + thread.join(); + } + +} diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java new file mode 100644 index 00000000000..967441ce250 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.lasterrorsholder; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.Server; +import com.yahoo.text.Utf8; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringWriter; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.time.Duration; +import java.time.Instant; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class LastErrorsHolderTestCase { + private static final int serverPort = 18324; + private static final int lastErrorsHolderPort = 18326; + Server server; + Thread serverThread; + LastErrorsHolder lastErrorsHolder; + + @Before + public void setUp() throws InterruptedException, IOException { + server = Server.getInstance(); + server.initialize(serverPort); + serverThread = new Thread(server); + serverThread.start(); + lastErrorsHolder = new LastErrorsHolder(lastErrorsHolderPort); + } + + @After + public void tearDown() throws InterruptedException { + if (serverThread!=null) { + serverThread.interrupt(); + serverThread.join(); + } + if (lastErrorsHolder !=null) lastErrorsHolder.close(); + } + + public String connectAndGetLogMessages() throws InterruptedException, IOException { + SocketChannel socket = null; + Instant start = Instant.now(); + while (Instant.now().isBefore(start.plus(Duration.ofMinutes(1)))) { + try { + socket = SocketChannel.open(new InetSocketAddress(lastErrorsHolderPort)); + break; + } catch (Exception e) { + Thread.sleep(100); + } + } + if (socket == null) { + throw new RuntimeException("Could not connect to server"); + } + + ByteBuffer buf = ByteBuffer.allocateDirect(10000); + int bytesRead = socket.read(buf); + byte[] bytes = new byte[bytesRead]; + buf.position(0); + buf.get(bytes); + socket.close(); + + return Utf8.toString(bytes); + } + + + @Test + public void testLastErrorsHolder() throws IOException, InvalidLogFormatException, InterruptedException { + LastErrorsHolder.Message logMessage1 = new LastErrorsHolder.Message(1433996283, "host1.yahoo.com", "container", LogLevel.ERROR.getName(), "foo"); + LastErrorsHolder.Message logMessage2 = new LastErrorsHolder.Message(1433996284, "host2.yahoo.com", "container", LogLevel.ERROR.getName(), "bar"); + LastErrorsHolder.Message logMessage3 = new LastErrorsHolder.Message(1433996285, "host2.yahoo.com", "container", LogLevel.INFO.getName(), "bar"); + + LastErrorsHolder.Messages messages = new LastErrorsHolder.Messages(); + + // No log messages yet + String logs = connectAndGetLogMessages(); + final ObjectMapper mapper = new ObjectMapper(); + StringWriter stringWriter = new StringWriter(); + mapper.writeValue(stringWriter, messages); + assertThat(logs, is(stringWriter.toString())); + + // Three messages, one is at level INFO + lastErrorsHolder.doHandle(createLogMessage(logMessage1)); + lastErrorsHolder.doHandle(createLogMessage(logMessage2)); + lastErrorsHolder.doHandle(createLogMessage(logMessage3)); + messages = new LastErrorsHolder.Messages(); + messages.addMessage(logMessage1); + messages.addMessage(logMessage2); + messages.setNumberOfErrors(2); + // Not adding logMessage3, since it is at level INFO + + logs = connectAndGetLogMessages(); + stringWriter = new StringWriter(); + mapper.writeValue(stringWriter, messages); + assertThat(logs, is(stringWriter.toString())); + } + + private LogMessage createLogMessage(LastErrorsHolder.Message message) throws InvalidLogFormatException { + return createLogMessage(message.getTime(), message.getHostname(), message.getService(), message.getLogLevel(), message.getMessage()); + } + + private LogMessage createLogMessage(long time, String hostname, String service, String logLevel, String message) throws InvalidLogFormatException { + return LogMessage.parseNativeFormat(String.format("%d\t%s\t1/1\t%s\tcomponent\t%s\t%s", time, hostname, service, logLevel.toLowerCase(), message)); + } + +} diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java new file mode 100644 index 00000000000..686c8a8a089 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java @@ -0,0 +1,122 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.logmetrics.test; + +import java.util.Map; + +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.handlers.logmetrics.LogMetricsHandler; +import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin; +import com.yahoo.plugin.SystemPropertyConfig; + +import org.junit.*; +import static org.junit.Assert.*; + +/** + * @author Harald Musum + */ +public class LogMetricsTestCase { + // Some of the tests depend upon the number of messages for a + // host, log level etc. to succeed, so you may have update the + // tests if you change something in mStrings. config, debug and + // spam are filtered out and not handled. + private static final String[] mStrings = { + "1095159244.095\thostA\t1/2\tservice\tcomponent\tconfig\tpayload1", + "1095206399.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload2", + "1095206400.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload3", + "1095206401.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload4", + "1095206402.000\thostA\t1/2\tservice\tcomponent\twarning\tpayload5", + "1095206403.000\thostA\t1/2\tservice\tcomponent\terror\tpayload6", + "1095206404.000\thostB\t1/2\tservice\tcomponent\tinfo\tpayload7", + "1095206405.000\thostB\t1/2\tservice\tcomponent\tfatal\tpayload8", + "1095206406.000\thostB\t1/2\tservice\tcomponent\tdebug\tpayload9", + }; + + private static final LogMessage[] msg = new LogMessage[mStrings.length]; + + static { + try { + for (int i = 0; i < mStrings.length; i++) { + msg[i] = LogMessage.parseNativeFormat(mStrings[i]); + } + } catch (InvalidLogFormatException e) { + throw new RuntimeException(e); + } + } + + /** + * Log some messages to the handler and verify that they are + * counted by the handler and that the count for each level is + * correct. + */ + @Test + public void testLevelCountTotal () throws java.io.IOException, InvalidLogFormatException { + LogMetricsHandler a = new LogMetricsHandler(); + + for (LogMessage aMsg : msg) { + a.handle(aMsg); + } + + long count = a.getMetricsCount(); + a.close(); + // Not all messages are processes (debug and spam) + assertEquals(count, 7); + } + + + + /** + * Log some messages to the handler and verify that they are + * counted by the handler and that the count for each level is + * correct (the count for each host is summed into one count for + * each level). + */ + @Test + public void testLevelCountAggregated () throws java.io.IOException, InvalidLogFormatException { + LogMetricsHandler a = new LogMetricsHandler(); + + for (LogMessage aMsg : msg) { + a.handle(aMsg); + } + + Map levelCount = a.getMetricsPerLevel(); + assertEquals(levelCount.entrySet().size(), 5); + for (Map.Entry entry : levelCount.entrySet()) { + String key = entry.getKey(); + if (key.equals("config")) { + assertEquals(entry.getValue(), new Long(1)); + } else if (key.equals("info")) { + assertEquals(entry.getValue(), new Long(4)); + } else if (key.equals("warning")) { + assertEquals(entry.getValue(), new Long(1)); + } else if (key.equals("severe")) { + assertEquals(entry.getValue(), new Long(0)); + } else if (key.equals("error")) { + assertEquals(entry.getValue(), new Long(1)); + } else if (key.equals("fatal")) { + assertEquals(entry.getValue(), new Long(1)); + } else if (key.equals("debug")) { + assertEquals(entry.getValue(), new Long(0)); // always 0 + } + } + a.close(); + } + + @Test + public void testLogMetricsPlugin() { + LogMetricsPlugin lp = new LogMetricsPlugin(); + try { + lp.shutdownPlugin(); + fail("Shutdown before init didn't throw."); + } catch (Exception e) { + } + lp.initPlugin(new SystemPropertyConfig("test")); + try { + lp.initPlugin(new SystemPropertyConfig("test")); + fail("Multiple init didn't throw."); + } catch (Exception e) { + } + lp.shutdownPlugin(); + } + +} diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java new file mode 100644 index 00000000000..b0d846a2965 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* -*- c-basic-offset: 4 -*- + * + * $Id$ + * + */ +package com.yahoo.logserver.handlers.replicator; + +import java.nio.ByteBuffer; +import com.yahoo.logserver.handlers.replicator.FormattedBufferCache; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.formatter.LogFormatter; +import com.yahoo.logserver.formatter.LogFormatterManager; +import com.yahoo.logserver.test.MockLogEntries; + +import org.junit.*; +import static org.junit.Assert.*; + +/** + * Unit tests for thge LogMessage class. + * + * @author Bjorn Borud + */ +public class FormattedBufferCacheTestCase { + + @Test + public void testCache () { + LogMessage msgs[] = MockLogEntries.getMessages(); + FormattedBufferCache cache = new FormattedBufferCache(); + String n[] = LogFormatterManager.getFormatterNames(); + for (int i = 0; i < n.length; i++) { + LogFormatter f = LogFormatterManager.getLogFormatter(n[i]); + for (int j = 0; j < msgs.length; j++) { + ByteBuffer bb = cache.getFormatted(msgs[j], f); + assertNotNull(bb); + } + } + + assertTrue(cache.getUnderlyingMapOnlyForTesting().size() > 0); + cache.reset(); + assertTrue(cache.getUnderlyingMapOnlyForTesting().size() == 0); + } +} diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java new file mode 100644 index 00000000000..dbb0b7817f6 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.replicator; + +import static org.junit.Assert.*; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.plugin.SystemPropertyConfig; + +public class ReplicatorPluginTestCase { + + ReplicatorPlugin rp; + + @Before + public void setUp() { + rp = new ReplicatorPlugin(); + } + + @After + public void tearDown() { + if (rp!=null) { + rp.shutdownPlugin(); + } + } + + @Test + public void testReplicatorPlugin() { + System.setProperty("replicatorplugin.test.port", "18325"); + try { + rp.shutdownPlugin(); + fail("Shutdown before init didn't throw."); + } catch (Exception e) { + } + rp.initPlugin(new SystemPropertyConfig("replicatorplugin.test.")); + try { + rp.initPlugin(new SystemPropertyConfig("test")); + fail("Multiple init didn't throw."); + } catch (Exception e) { + } + } +} diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java new file mode 100644 index 00000000000..37f25e47a1c --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.handlers.replicator; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.yahoo.io.Listener; +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.Server; +import com.yahoo.logserver.filter.LogFilter; +import com.yahoo.logserver.filter.LogFilterManager; +import com.yahoo.logserver.formatter.LogFormatterManager; + +public class ReplicatorTestCase { + Server server; + Thread serverThread; + Replicator replicator; + private SocketChannel socket; + private Listener listener; + private ReplicatorConnection conn; + + @Before + public void setUp() throws InterruptedException, IOException { + server = Server.getInstance(); + server.initialize(18321); + serverThread = new Thread(server); + serverThread.start(); + long start = System.currentTimeMillis(); + long timeout = 60000; + while (System.currentTimeMillis() < (start + timeout)) { + try { + socket = SocketChannel.open(new InetSocketAddress(18321)); + break; + } catch (Exception e) { + Thread.sleep(100); + } + } + listener = new Listener("test"); + replicator = new Replicator(18323); + conn = (ReplicatorConnection) replicator.newConnection(socket, listener); + } + + @After + public void tearDown() throws InterruptedException { + if (serverThread!=null) { + serverThread.interrupt(); + serverThread.join(); + } + if (replicator!=null) replicator.close(); + } + + @Test + public void testReplicator() throws IOException, InvalidLogFormatException { + LogMessage msg = LogMessage. + parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\tevent\tfoo"); + assertFalse(conn.isLoggable(msg)); // Default all muted + conn.onUse("system.all"); + assertTrue(conn.isLoggable(msg)); + assertTrue(conn.getTotalBytesWritten()>50); // Should be in this ballpark + conn.onCommand("use system.mute"); + assertFalse(conn.isLoggable(msg)); + assertEquals("system.mute", conn.getLogFilterName()); + replicator.doHandle(msg); + conn.read(); + assertEquals(2, conn.getNumHandled()); + conn.onFormatter("system.textformatter"); + assertEquals(conn.formatter, LogFormatterManager.getLogFormatter("system.textformatter")); + conn.onCommand("formatter system.nullformatter"); + assertEquals(conn.formatter, LogFormatterManager.getLogFormatter("system.nullformatter")); + assertEquals(4, conn.getNumHandled()); + conn.onList(); + assertEquals(11, conn.getNumHandled()); // 6 filters + start/stop msg + conn.onCommand("list"); + assertEquals(18, conn.getNumHandled()); // 6 filters + start/stop msg + conn.onListFormatters(); + assertEquals(22, conn.getNumHandled()); // 4 formatters + conn.onCommand("listformatters"); + assertEquals(26, conn.getNumHandled()); // 4 formatters + + conn.onStats(); + assertEquals(27, conn.getNumHandled()); // 1 line + conn.onCommand("stats"); + assertEquals(28, conn.getNumHandled()); // 1 line + conn.enqueue(ByteBuffer.wrap("baz".getBytes())); + assertEquals(29, conn.getNumHandled()); // 1 line + conn.onCommand("ping"); + assertEquals(30, conn.getNumHandled()); // 1 line + conn.onCommand("quit"); + assertEquals(31, conn.getNumHandled()); // 1 line + assertEquals(0, conn.getNumDropped()); + assertEquals(0, conn.getNumQueued()); + LogFilterManager.addLogFilter("test.onlyerror", new LogFilter() { + @Override + public boolean isLoggable(LogMessage msg) { + return msg.getLevel().equals(LogLevel.ERROR); + } + + @Override + public String description() { + return "Only error"; + } + }); + conn.onUse("test.onlyerror"); + assertFalse(conn.isLoggable(LogMessage. + parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\tdebug\tfoo"))); + assertTrue(conn.isLoggable(LogMessage. + parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar"))); + assertEquals(conn.selectOps(), 1); + assertEquals(conn.description(), "Only error"); + conn.setFilter(null); + assertTrue(conn.isLoggable(LogMessage. + parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar"))); + assertEquals(conn.description(), "No filter defined"); + assertEquals(conn.getRemoteHost(), "0.0.0.0"); + conn.onFormatter("nonexistant"); + assertEquals(conn.formatter, LogFormatterManager.getLogFormatter("system.nullformatter")); // unchanged + conn.onUse("nonexistant"); + assertTrue(conn.isLoggable(LogMessage. + parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar"))); + conn.close(); + } + +} diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/status/.gitignore b/logserver/src/test/java/com/yahoo/logserver/handlers/status/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/logserver/src/test/java/com/yahoo/logserver/httphandlers/.gitignore b/logserver/src/test/java/com/yahoo/logserver/httphandlers/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java b/logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java new file mode 100644 index 00000000000..5f0897187c4 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.net.control.test; + +import com.yahoo.logserver.net.control.Levels; +import com.yahoo.logserver.net.control.State; + +import org.junit.*; +import static org.junit.Assert.*; + +/** + * @author Bjorn Borud + */ +public class LevelsTestCase { + /** + * Make sure the parsing works + */ + @Test + public void testParser() { + Levels levels = Levels.parse("fatal=forward,error=forward,warning=noforward"); + assertNotNull(levels); + assertSame(State.FORWARD, levels.getLevelState("fatal")); + assertSame(State.FORWARD, levels.getLevelState("error")); + assertSame(State.NOFORWARD, levels.getLevelState("warning")); + } + + /** + * Ensure that the Levels are in a known default state. A Levels + * object that was newly created is in a defined default state + * which has been deemed to be "reasonable". In general, it will + * specify forwarding all log levels except "debug" and it will + * specify "spam" to be turned off. + * + */ + @Test + public void testDefaultLevels () { + Levels levels = new Levels(); + assertSame(State.FORWARD, levels.getLevelState("event")); + assertSame(State.FORWARD, levels.getLevelState("fatal")); + assertSame(State.FORWARD, levels.getLevelState("error")); + assertSame(State.FORWARD, levels.getLevelState("warning")); + assertSame(State.FORWARD, levels.getLevelState("info")); + assertSame(State.FORWARD, levels.getLevelState("config")); + assertSame(State.NOFORWARD, levels.getLevelState("debug")); + assertSame(State.NOFORWARD, levels.getLevelState("spam")); + } + + /** + * This test also documents/verifies the default behavior + * of the Levels class. + * + */ + @Test + public void testToString() { + String in = "fatal=forward,error=forward,warning=noforward"; + String out = "event=forward,fatal=forward,error=forward,warning=noforward,info=forward,config=forward,debug=noforward,spam=noforward"; + Levels levels = Levels.parse(in); + assertEquals(out, levels.toString()); + + } + + /** + * Clone testing + */ + @Test + public void testClone () { + Levels l1 = Levels.parse("error=noforward"); + assertEquals(l1.toString(), l1.clone().toString()); + assertSame(State.NOFORWARD, l1.getLevelState("error")); + assertSame(State.NOFORWARD, ((Levels)l1.clone()).getLevelState("error")); + assertSame(l1.getLevelState("error"), + ((Levels)l1.clone()).getLevelState("error")); + } + + /** + * test parser + */ + @Test + public void testUpdateLevels () { + Levels l1 = Levels.parse("error=noforward"); + assertSame(State.NOFORWARD, l1.getLevelState("error")); + + // should be unaffected + assertSame(State.FORWARD, l1.getLevelState("warning")); + + // update and check that the update worked + l1.updateLevels("error=noforward"); + assertSame(State.NOFORWARD, l1.getLevelState("error")); + + // should be unaffected + assertSame(State.FORWARD, l1.getLevelState("warning")); + } +} diff --git a/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java b/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java new file mode 100644 index 00000000000..06f35567e9f --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java @@ -0,0 +1,211 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.net.test; + +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.LogDispatcher; +import com.yahoo.logserver.handlers.AbstractLogHandler; +import com.yahoo.logserver.net.LogConnection; +import com.yahoo.logserver.test.MockLogEntries; +import org.junit.Test; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.nio.charset.Charset; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; +import java.util.logging.Logger; + +import static org.junit.Assert.*; + +/** + * Unit tests for the log connection class. Wow is this too + * complex! + * + * @author Bjorn Borud + */ +public class LogConnectionTestCase { + private static final Logger log = Logger.getLogger(LogConnectionTestCase.class.getName()); + + private static final int PROBING_RANGE_START = 41352; + private static final Charset charset = Charset.forName("utf-8"); + private static final ByteBuffer bigBuffer; + private int port; + + static { + StringBuilder sb = new StringBuilder(LogConnection.READBUFFER_SIZE * 6) + .append(MockLogEntries.getMessages()[0].toString()) + .append(MockLogEntries.getMessages()[1].toString()) + .append(MockLogEntries.getMessages()[2].toString()); + + // get a valid log message prefix + String prefix = MockLogEntries.getMessages()[2].toString(); + prefix = prefix.substring(0, prefix.length() - 1); + sb.append(prefix); + + // fill up the remainding buffer with rubbish to make + // it too long + for (int i = 0; i < (LogConnection.READBUFFER_SIZE * 3); i++) { + sb.append("a"); + } + sb.append('\n'); + sb.append(MockLogEntries.getMessages()[2].toString()); + bigBuffer = charset.encode(sb.toString()); + } + + /** + * this code is nothing short of completely hideous. the exception + * handling is awful and the code is messy, but it should be a fairly + * efficient and robust way of testing buffer overflow conditions. + */ + @Test + public void testOverflow() { + final CyclicBarrier barrier = new CyclicBarrier(2); + + + // this inner class is used to define a mock handler + // which will help us examine the messages actually + // handled. + class MockHandler extends AbstractLogHandler { + private final List messages = new LinkedList(); + + public boolean doHandle(LogMessage msg) { + messages.add(msg); + return true; + } + + public List getMessages() { + return messages; + } + + public void flush() { + } + + public void close() { + } + + public String toString() { + return null; + } + } + + Thread serverThread = new Thread() { + public void run() { + ServerSocketChannel ss = setUpListenSocket(); + if (ss == null) { + fail("unable to set up listen socket"); + return; + } + + setPort(ss.socket().getLocalPort()); + + // listen port is up now so we can trigger the barrier + try { + barrier.await(); + + while (!Thread.currentThread().isInterrupted()) { + SocketChannel s = ss.accept(); + pushBigBuffer(s); + s.close(); + } + } catch (BrokenBarrierException e) { + fail(e.getMessage()); + return; + } catch (InterruptedException | java.nio.channels.ClosedByInterruptException e) { + return; + } catch (IOException e) { + log.log(LogLevel.ERROR, "argh", e); + fail(); + return; + } + + } + }; + serverThread.start(); + assertTrue(serverThread.isAlive()); + + try { + barrier.await(); + } catch (BrokenBarrierException e) { + fail(e.getMessage()); + return; + } catch (InterruptedException e) { + return; + } + + SocketChannel sock; + try { + sock = SocketChannel.open(new InetSocketAddress("localhost", port)); + } catch (IOException e) { + fail(e.getMessage()); + return; + } + + LogDispatcher dispatcher = new LogDispatcher(); + MockHandler mock = new MockHandler(); + assertTrue(mock.getName().endsWith("MockHandler")); + dispatcher.registerLogHandler(mock); + LogConnection logConnection = + new LogConnection(sock, null, dispatcher, null); + + try { + for (int i = 0; i < 100; i++) { + logConnection.read(); + } + } catch (java.nio.channels.ClosedChannelException e) { + // ignore, this is normal + } catch (IOException e) { + log.log(LogLevel.ERROR, "error during reading", e); + } + + // there should be 5 messages + assertEquals(5, mock.getMessages().size()); + assertEquals(5, mock.getCount()); + + // the 4'th message should be long + String m = (mock.getMessages().get(3)).getPayload(); + assertTrue(m.length() > 10000); + + serverThread.interrupt(); + try { + serverThread.join(); + assertTrue(true); + } catch (InterruptedException e) { + fail(); + } + } + + private void pushBigBuffer(SocketChannel socket) { + try { + ByteBuffer slice = bigBuffer.slice(); + while (slice.hasRemaining()) { + @SuppressWarnings("unused") + int ret = socket.write(slice); + } + } catch (java.io.IOException e) { + + } + } + + private void setPort(int port) { + this.port = port; + } + + private ServerSocketChannel setUpListenSocket() { + int p = 18327; + try { + ServerSocketChannel s = ServerSocketChannel.open(); + s.socket().setReuseAddress(true); + s.socket().bind(new InetSocketAddress("127.0.0.1", p)); + return s; + } catch (Exception e) { + fail(e.getMessage()); + } + return null; + } +} diff --git a/logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java b/logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java new file mode 100644 index 00000000000..37ad625247d --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.test; + +import java.util.ArrayList; +import java.util.List; + +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.logserver.LogDispatcher; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.handlers.LogHandler; + +import org.junit.*; +import static org.junit.Assert.*; + +/** + * Unit tests for thge LogMessage class. + * + * @author Bjorn Borud + */ +public class LogDispatcherTestCase { + private static LogMessage sample1; + private static LogMessage sample2; + static { + try { + sample1 = LogMessage.parseNativeFormat("1096639280.524133 malfunction 26851 - logtest info Starting up, called as ./log/logtest"); + sample2 = LogMessage.parseNativeFormat("1096639280.524133 malfunction 26851 - logtest info More crap"); + } + catch (InvalidLogFormatException e) { + assertTrue(false); + } + } + + public static class MockHandler implements LogHandler { + public final List messages= new ArrayList(5); + public int flushCalled = 0; + public int closeCalled = 0; + + public void handle (LogMessage msg) { + messages.add(msg); + } + + public void handle (List messages) { + for (LogMessage lm : messages) { + handle(lm); + } + } + + public void flush () { + flushCalled++; + } + public void close () { + closeCalled++; + } + public String getName () { + return MockHandler.class.getName(); + } + } + + @Test + public void testLogDispatcherBatchMode() { + MockHandler handler = new MockHandler(); + LogDispatcher dispatcher = new LogDispatcher(); + dispatcher.setBatchedMode(true); + dispatcher.registerLogHandler(handler); + assertEquals(0, dispatcher.getMessageCount()); + dispatcher.handle(sample1); + assertEquals(1, dispatcher.getMessageCount()); + dispatcher.handle(sample2); + assertEquals(2, dispatcher.getMessageCount()); + assertEquals(0, handler.messages.size()); + dispatcher.flush(); + assertEquals(2, handler.messages.size()); + } + + @Test + public void testTestLogHandlerRegistration () { + MockHandler handler = new MockHandler(); + LogDispatcher dispatcher = new LogDispatcher(); + dispatcher.registerLogHandler(handler); + + assertNotNull(dispatcher.getLogHandlers()); + LogHandler[] handlers = dispatcher.getLogHandlers(); + assertNotNull(handlers); + assertEquals(1, handlers.length); + assertTrue(handler == handlers[0]); + } + + @Test + public void testMessageCount () { + MockHandler handler = new MockHandler(); + LogDispatcher dispatcher = new LogDispatcher(); + dispatcher.registerLogHandler(handler); + assertEquals(0, dispatcher.getMessageCount()); + + dispatcher.handle(sample1); + assertEquals(1, dispatcher.getMessageCount()); + + dispatcher.handle(sample2); + assertEquals(2, dispatcher.getMessageCount()); + + assertEquals(2, handler.messages.size()); + } + + @Test + public void testVerifyMessages () { + MockHandler handler = new MockHandler(); + LogDispatcher dispatcher = new LogDispatcher(); + dispatcher.registerLogHandler(handler); + + dispatcher.handle(sample1); + dispatcher.handle(sample2); + + assertTrue(sample1 == handler.messages.get(0)); + assertTrue(sample2 == handler.messages.get(1)); + } + + // TODO: this test makes very little sense until we refactor a bit + @Test + public void testClose () { + MockHandler handler = new MockHandler(); + LogDispatcher dispatcher = new LogDispatcher(); + dispatcher.registerLogHandler(handler); + + assertEquals(0, handler.flushCalled); + assertEquals(0, handler.closeCalled); + + dispatcher.close(); + assertEquals(0, handler.closeCalled); + } + +} diff --git a/logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java b/logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java new file mode 100644 index 00000000000..a9bb8084a7c --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.test; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.zip.GZIPInputStream; + +import com.yahoo.log.event.Event; +import com.yahoo.log.event.MalformedEventException; +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogMessage; + +import org.junit.*; +import static org.junit.Assert.*; + +/** + * Unit tests for thge LogMessage class. + * + * @author Bjorn Borud + */ +public class LogMessageTestCase { + + /** + * The original test was rubbish. We just test that parsing + * is okay here. The way we do it is to check that we have + * some log messages available from the MockLogEntries class. + * If there are none the parsing failed. + */ + @Test + public void testLogParsing () + { + assertTrue(MockLogEntries.getMessages().length > 0); + } + + /** + * Read in some events and make sure we are able to identify + * them as such. + */ + @Test + public void testEvents () throws IOException { + String eventfile = "src/test/files/event.txt.gz"; + BufferedReader br = new BufferedReader( + new InputStreamReader( + new GZIPInputStream( + new FileInputStream(eventfile)))); + + for (String line = br.readLine(); line != null; line = br.readLine()) { + try { + LogMessage m = LogMessage.parseNativeFormat(line); + try { + Event event = m.getEvent(); + assertNotNull(event); + } catch (MalformedEventException evx) { + fail(); + } + } catch (InvalidLogFormatException e) { + fail(); + } + } + } +} + diff --git a/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java b/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java new file mode 100644 index 00000000000..18cf8afb8a0 --- /dev/null +++ b/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.logserver.test; + +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogMessage; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; + +/** + * This class is used to gain access to a bunch of log entries + * so we can use the same log messages in several different tests + * + * @author Bjorn Borud + */ +public class MockLogEntries { + private static final MockLogEntries instance = new MockLogEntries(); + + private final LogMessage[] messages; + + /** + * Private constructor which reads the log messages and builds + * an array of LogMessage entries. + */ + private MockLogEntries () { + List msgs = new LinkedList(); + try { + String name = "src/test/files/logEntries.txt"; + BufferedReader br = new BufferedReader(new FileReader(name)); + for (String line = br.readLine(); line != null; line = br.readLine()) { + LogMessage m = LogMessage.parseNativeFormat(line); + msgs.add(m); + } + } catch (InvalidLogFormatException | IOException e) { + // do nothing + } + + LogMessage[] m = new LogMessage[msgs.size()]; + msgs.toArray(m); + messages = m; + } + + /** + * Return the LogMessage instances we've constructed from the + * stored log messages we have. + */ + public static LogMessage[] getMessages() { + return instance.messages; + } +} -- cgit v1.2.3