diff options
20 files changed, 508 insertions, 338 deletions
diff --git a/logserver/pom.xml b/logserver/pom.xml index a813558580c..ebee1296f4b 100644 --- a/logserver/pom.xml +++ b/logserver/pom.xml @@ -46,6 +46,12 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + <version>2.25.1</version> + </dependency> </dependencies> <build> <plugins> diff --git a/logserver/src/main/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethod.java b/logserver/src/main/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethod.java new file mode 100644 index 00000000000..9d234b73692 --- /dev/null +++ b/logserver/src/main/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethod.java @@ -0,0 +1,92 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.logserver.protocol; + +import com.yahoo.jrt.DataValue; +import com.yahoo.jrt.ErrorCode; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Int8Value; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.logserver.LogDispatcher; + +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * RPC method that archives incoming log messages + * + * @author bjorncs + */ +public class ArchiveLogMessagesMethod { + + static final String METHOD_NAME = "vespa.logserver.archiveLogMessages"; + + private static final Logger log = Logger.getLogger(ArchiveLogMessagesMethod.class.getName()); + + private final Executor executor = Executors.newSingleThreadExecutor(); + private final LogDispatcher logDispatcher; + private final Method method; + + public ArchiveLogMessagesMethod(LogDispatcher logDispatcher) { + this.logDispatcher = logDispatcher; + this.method = new Method(METHOD_NAME, "bix", "bix", this::log) + .methodDesc("Archive log messages") + .paramDesc(0, "compressionType", "Compression type (0=raw)") + .paramDesc(1, "uncompressedSize", "Uncompressed size") + .paramDesc(2, "logRequest", "Log request encoded with protobuf") + .returnDesc(0, "compressionType", "Compression type (0=raw)") + .returnDesc(1, "uncompressedSize", "Uncompressed size") + .returnDesc(2, "logResponse", "Log response encoded with protobuf"); + } + + public Method methodDefinition() { + return method; + } + + private void log(Request rpcRequest) { + rpcRequest.detach(); + executor.execute(new ArchiveLogMessagesTask(rpcRequest, logDispatcher)); + } + + private static class ArchiveLogMessagesTask implements Runnable { + final Request rpcRequest; + final LogDispatcher logDispatcher; + + ArchiveLogMessagesTask(Request rpcRequest, LogDispatcher logDispatcher) { + this.rpcRequest = rpcRequest; + this.logDispatcher = logDispatcher; + } + + @Override + public void run() { + try { + byte compressionType = rpcRequest.parameters().get(0).asInt8(); + if (compressionType != 0) { + rpcRequest.setError(ErrorCode.METHOD_FAILED, "Invalid compression type: " + compressionType); + rpcRequest.returnRequest(); + return; + } + int uncompressedSize = rpcRequest.parameters().get(1).asInt32(); + byte[] logRequestPayload = rpcRequest.parameters().get(2).asData(); + if (uncompressedSize != logRequestPayload.length) { + rpcRequest.setError(ErrorCode.METHOD_FAILED, String.format("Invalid uncompressed size: got %d while data is of size %d ", uncompressedSize, logRequestPayload.length)); + rpcRequest.returnRequest(); + return; + } + logDispatcher.handle(ProtobufSerialization.fromLogRequest(logRequestPayload)); + rpcRequest.returnValues().add(new Int8Value((byte)0)); + byte[] responsePayload = ProtobufSerialization.toLogResponse(); + rpcRequest.returnValues().add(new Int32Value(responsePayload.length)); + rpcRequest.returnValues().add(new DataValue(responsePayload)); + rpcRequest.returnRequest(); + } catch (Exception e) { + String errorMessage = "Failed to handle log request: " + e.getMessage(); + log.log(Level.WARNING, e, () -> errorMessage); + rpcRequest.setError(ErrorCode.METHOD_FAILED, errorMessage); + rpcRequest.returnRequest(); + } + } + } +} diff --git a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java new file mode 100644 index 00000000000..9065b47fa52 --- /dev/null +++ b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java @@ -0,0 +1,130 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.logserver.protocol; + +import ai.vespa.logserver.protocol.protobuf.LogProtocol; +import com.google.protobuf.InvalidProtocolBufferException; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; + +import java.time.Instant; +import java.util.List; +import java.util.logging.Level; + +import static java.util.stream.Collectors.toList; + +/** + * Utility class for serialization of log requests and responses. + * + * @author bjorncs + */ +class ProtobufSerialization { + + private ProtobufSerialization() {} + + static List<LogMessage> fromLogRequest(byte[] logRequestPayload) { + try { + LogProtocol.LogRequest logRequest = LogProtocol.LogRequest.parseFrom(logRequestPayload); + return logRequest.getLogMessagesList().stream() + .map(ProtobufSerialization::fromLogRequest) + .collect(toList()); + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException("Unable to parse log request: " + e.getMessage(), e); + } + } + + static byte[] toLogRequest(List<LogMessage> logMessages) { + LogProtocol.LogRequest.Builder builder = LogProtocol.LogRequest.newBuilder(); + for (LogMessage logMessage : logMessages) { + builder.addLogMessages(toLogRequestMessage(logMessage)); + } + return builder.build().toByteArray(); + } + + static byte[] toLogResponse() { + return LogProtocol.LogResponse.newBuilder().build().toByteArray(); + } + + static void fromLogResponse(byte[] logResponsePayload) { + try { + LogProtocol.LogResponse.parseFrom(logResponsePayload); // log response is empty + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException("Unable to parse log response: " + e.getMessage(), e); + } + } + + private static LogMessage fromLogRequest(LogProtocol.LogMessage message) { + return LogMessage.of( + Instant.ofEpochSecond(0, message.getTimeNanos()), + message.getHostname(), + message.getProcessId(), + message.getThreadId(), + message.getService(), + message.getComponent(), + fromLogMessageLevel(message.getLevel()), + message.getPayload()); + } + + private static LogProtocol.LogMessage toLogRequestMessage(LogMessage logMessage) { + Instant timestamp = logMessage.getTimestamp(); + long timestampNanos = timestamp.getEpochSecond() * 1_000_000_000L + timestamp.getNano(); + return LogProtocol.LogMessage.newBuilder() + .setTimeNanos(timestampNanos) + .setHostname(logMessage.getHost()) + .setProcessId((int) logMessage.getProcessId()) + .setThreadId((int) logMessage.getThreadId().orElse(0)) + .setService(logMessage.getService()) + .setComponent(logMessage.getComponent()) + .setLevel(toLogMessageLevel(logMessage.getLevel())) + .setPayload(logMessage.getPayload()) + .build(); + } + + private static Level fromLogMessageLevel(LogProtocol.LogMessage.Level level) { + switch (level) { + case FATAL: + return LogLevel.FATAL; + case ERROR: + return LogLevel.ERROR; + case WARNING: + return LogLevel.WARNING; + case CONFIG: + return LogLevel.CONFIG; + case INFO: + return LogLevel.INFO; + case EVENT: + return LogLevel.EVENT; + case DEBUG: + return LogLevel.DEBUG; + case SPAM: + return LogLevel.SPAM; + case UNKNOWN: + case UNRECOGNIZED: + default: + return LogLevel.UNKNOWN; + } + } + + private static LogProtocol.LogMessage.Level toLogMessageLevel(Level level) { + Level vespaLevel = LogLevel.getVespaLogLevel(level); + if (vespaLevel.equals(LogLevel.FATAL)) { + return LogProtocol.LogMessage.Level.FATAL; + } else if (vespaLevel.equals(LogLevel.ERROR)) { + return LogProtocol.LogMessage.Level.ERROR; + } else if (vespaLevel.equals(LogLevel.WARNING)) { + return LogProtocol.LogMessage.Level.WARNING; + } else if (vespaLevel.equals(LogLevel.CONFIG)) { + return LogProtocol.LogMessage.Level.CONFIG; + } else if (vespaLevel.equals(LogLevel.INFO)) { + return LogProtocol.LogMessage.Level.INFO; + } else if (vespaLevel.equals(LogLevel.EVENT)) { + return LogProtocol.LogMessage.Level.EVENT; + } else if (vespaLevel.equals(LogLevel.DEBUG)) { + return LogProtocol.LogMessage.Level.DEBUG; + } else if (vespaLevel.equals(LogLevel.SPAM)) { + return LogProtocol.LogMessage.Level.SPAM; + } else { + return LogProtocol.LogMessage.Level.UNKNOWN; + } + } + +} diff --git a/logserver/src/main/java/ai/vespa/logserver/protocol/RpcServer.java b/logserver/src/main/java/ai/vespa/logserver/protocol/RpcServer.java new file mode 100644 index 00000000000..b3860da6fb7 --- /dev/null +++ b/logserver/src/main/java/ai/vespa/logserver/protocol/RpcServer.java @@ -0,0 +1,49 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.logserver.protocol; + +import com.yahoo.jrt.Acceptor; +import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; + +/** + * A JRT based RPC server for handling log requests + * + * @author bjorncs + */ +public class RpcServer implements AutoCloseable { + + private final Supervisor supervisor = new Supervisor(new Transport()); + private final int listenPort; + private Acceptor acceptor; + + public RpcServer(int listenPort) { + this.listenPort = listenPort; + } + + public void addMethod(Method method) { + supervisor.addMethod(method); + } + + public void start() { + try { + acceptor = supervisor.listen(new Spec(listenPort)); + } catch (ListenFailedException e) { + throw new RuntimeException(e); + } + } + + int listenPort() { + return acceptor.port(); + } + + @Override + public void close() { + if (acceptor != null) { + acceptor.shutdown().join(); + } + supervisor.transport().shutdown().join(); + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/Server.java b/logserver/src/main/java/com/yahoo/logserver/Server.java index db01444ca67..68ab8be2fba 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Server.java +++ b/logserver/src/main/java/com/yahoo/logserver/Server.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver; +import ai.vespa.logserver.protocol.ArchiveLogMessagesMethod; +import ai.vespa.logserver.protocol.RpcServer; import com.yahoo.io.FatalErrorHandler; import com.yahoo.io.Listener; import com.yahoo.log.LogLevel; @@ -39,6 +41,7 @@ public class Server implements Runnable { LogSetup.initVespaLogging("ADM"); } + private static final int DEFAULT_RPC_LISTEN_PORT = 19080; // the port is a String because we want to use it as the default // value of a System.getProperty(). private static final String LISTEN_PORT = "19081"; @@ -46,6 +49,7 @@ public class Server implements Runnable { private int listenPort; private Listener listener; private final LogDispatcher dispatch; + private RpcServer rpcServer; private final boolean isInitialized; @@ -108,8 +112,9 @@ public class Server implements Runnable { * * @param listenPort The port on which the logserver accepts log * messages. + * @param rpcListenPort */ - public void initialize(int listenPort) { + public void initialize(int listenPort, int rpcListenPort) { if (isInitialized) { throw new IllegalStateException(APPNAME + " already initialized"); } @@ -123,6 +128,8 @@ public class Server implements Runnable { listener = new Listener(APPNAME); listener.addSelectLoopPostHook(dispatch); listener.setFatalErrorHandler(fatalErrorHandler); + rpcServer = new RpcServer(rpcListenPort); + rpcServer.addMethod(new ArchiveLogMessagesMethod(dispatch).methodDefinition()); } /** @@ -140,6 +147,8 @@ public class Server implements Runnable { log.fine("Starting listener..."); listener.start(); + log.fine("Starting rpc server..."); + rpcServer.start(); Event.started(APPNAME); try { listener.join(); @@ -164,6 +173,7 @@ public class Server implements Runnable { } } Event.stopping(APPNAME, "shutdown"); + rpcServer.close(); dispatch.close(); Event.stopped(APPNAME, 0, 0); System.exit(0); @@ -176,6 +186,7 @@ public class Server implements Runnable { static void help() { System.out.println(); System.out.println("System properties:"); + System.out.println(" - " + APPNAME + ".rpcListenPort (" + DEFAULT_RPC_LISTEN_PORT + ")"); System.out.println(" - " + APPNAME + ".listenport (" + LISTEN_PORT + ")"); System.out.println(" - " + APPNAME + ".queue.size (" + HandlerThread.DEFAULT_QUEUESIZE + ")"); System.out.println(); @@ -188,9 +199,10 @@ public class Server implements Runnable { } String portString = System.getProperty(APPNAME + ".listenport", LISTEN_PORT); + int rpcPort = Integer.parseInt(System.getProperty(APPNAME + ".rpcListenPort", Integer.toString(DEFAULT_RPC_LISTEN_PORT))); Server server = Server.getInstance(); server.setupSignalHandler(); - server.initialize(Integer.parseInt(portString)); + server.initialize(Integer.parseInt(portString), rpcPort); Thread t = new Thread(server, "logserver main"); t.start(); diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java deleted file mode 100644 index 03bb7787b65..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.formatter; - -import com.yahoo.log.LogMessage; - -/** - * This interface is analogous to the java.util.logging.Formatter - * interface. Classes implementing this interface should be - * <b>stateless/immutable if possible so formatters can be - * shared</b>. If it does have state it must not prevent - * concurrent use. - * - * @author Bjorn Borud - */ -public interface LogFormatter { - /** - * Format log message as a string. - * - * @param msg The log message - */ - String format(LogMessage msg); - - /** - * Returns a textual description of the formatter - */ - String description(); -} diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java deleted file mode 100644 index 1d0d29adb9e..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * $Id$ - * - */ - -package com.yahoo.logserver.formatter; - -import java.util.HashMap; -import java.util.Map; - -/** - * This singleton class implements a central registry of LogFormatter - * instances. - * - * @author Bjorn Borud - */ -public class LogFormatterManager { - private static final LogFormatterManager instance; - - static { - instance = new LogFormatterManager(); - instance.addLogFormatterInternal("system.textformatter", new TextFormatter()); - instance.addLogFormatterInternal("system.nullformatter", new NullFormatter()); - } - - private final Map<String, LogFormatter> logFormatters = new HashMap<String, LogFormatter>(); - - private LogFormatterManager() {} - - /** - * LogFormatter lookup function - * - * @param name The name of the LogFormatter to be looked up. - * @return Returns the LogFormatter associated with this name or - * <code>null</code> if not found. - */ - public static LogFormatter getLogFormatter(String name) { - synchronized (instance.logFormatters) { - return instance.logFormatters.get(name); - } - } - - /** - * Get the names of the defined formatters. - * - * @return Returns an array containing the names of formatters that - * have been registered. - */ - public static String[] getFormatterNames() { - synchronized (instance.logFormatters) { - String[] formatterNames = new String[instance.logFormatters.keySet().size()]; - instance.logFormatters.keySet().toArray(formatterNames); - return formatterNames; - } - } - - /** - * Internal method which takes care of the job of adding - * LogFormatter mappings but doesn't perform any of the checks - * performed by the public method for adding mappings. - */ - private void addLogFormatterInternal(String name, LogFormatter logFormatter) { - synchronized (logFormatters) { - logFormatters.put(name, logFormatter); - } - } - -} diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java deleted file mode 100644 index c419ce21272..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * $Id$ - * - */ - -package com.yahoo.logserver.formatter; - -import com.yahoo.log.LogMessage; - -/** - * This formatter doesn't really format anything. It just - * calls the LogMessage toString() method. This is kind of - * pointless and silly, but we include it for symmetry... - * or completeness....or...whatever. - * - * @author Bjorn Borud - */ -public class NullFormatter implements LogFormatter { - - public String format(LogMessage msg) { - return msg.toString(); - } - - public String description() { - return "Format message in native VESPA format"; - } - -} diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java deleted file mode 100644 index efc4a9898ed..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * $Id$ - * - */ - -package com.yahoo.logserver.formatter; - -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.TimeZone; - -import com.yahoo.log.LogMessage; - -/** - * Creates human-readable text representation of log message. - * - * @author Bjorn Borud - */ -public class TextFormatter implements LogFormatter { - static final SimpleDateFormat dateFormat; - - static { - dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - } - - public String format(LogMessage msg) { - StringBuffer sbuf = new StringBuffer(150); - sbuf.append(dateFormat.format(new Date(msg.getTime()))) - .append(" ") - .append(msg.getHost()) - .append(" ") - .append(msg.getThreadProcess()) - .append(" ") - .append(msg.getService()) - .append(" ") - .append(msg.getComponent()) - .append(" ") - .append(msg.getLevel().toString()) - .append(" ") - .append(msg.getPayload()) - .append("\n"); - - return sbuf.toString(); - } - - public String description() { - return "Format log-message as human readable text"; - } -} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index f1e1665a7d1..bf7911388dc 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -1,6 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.handlers.archive; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.filter.LogFilter; +import com.yahoo.logserver.filter.LogFilterManager; +import com.yahoo.logserver.handlers.AbstractLogHandler; + import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; @@ -11,13 +17,6 @@ import java.util.TimeZone; import java.util.logging.Level; import java.util.logging.Logger; -import com.yahoo.logserver.filter.LogFilter; -import com.yahoo.logserver.filter.LogFilterManager; - -import com.yahoo.log.LogLevel; -import com.yahoo.log.LogMessage; -import com.yahoo.logserver.handlers.AbstractLogHandler; - /** * This class implements a log handler which archives the incoming @@ -131,7 +130,7 @@ public class ArchiverHandler extends AbstractLogHandler { * Return the appropriate LogWriter given a log message. */ private synchronized LogWriter getLogWriter(LogMessage m) throws IOException { - Integer slot = dateHash(m.getTime()); + Integer slot = dateHash(m.getTimestamp().toEpochMilli()); LogWriter logWriter = logWriterLRUCache.get(slot); if (logWriter != null) { return logWriter; @@ -174,7 +173,7 @@ public class ArchiverHandler extends AbstractLogHandler { * XXX optimize! */ public String getPrefix(LogMessage msg) { - calendar.setTimeInMillis(msg.getTime()); + calendar.setTimeInMillis(msg.getTimestamp().toEpochMilli()); /* int year = calendar.get(Calendar.YEAR); int month = calendar.get(Calendar.MONTH) + 1; diff --git a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java new file mode 100644 index 00000000000..a30df6bb050 --- /dev/null +++ b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java @@ -0,0 +1,84 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.logserver.protocol; + +import com.yahoo.jrt.DataValue; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Int8Value; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.Transport; +import com.yahoo.jrt.Values; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogMessage; +import com.yahoo.logserver.LogDispatcher; +import org.junit.Test; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; + +import static ai.vespa.logserver.protocol.ProtobufSerialization.fromLogResponse; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +/** + * @author bjorncs + */ +public class ArchiveLogMessagesMethodTest { + + private static final LogMessage MESSAGE_1 = + LogMessage.of(Instant.EPOCH.plus(1000, ChronoUnit.DAYS), "localhost", 12, 3456, "my-service", "my-component", LogLevel.ERROR, "My error message"); + private static final LogMessage MESSAGE_2 = + LogMessage.of(Instant.EPOCH.plus(5005, ChronoUnit.DAYS), "localhost", 12, 6543, "my-service", "my-component", LogLevel.INFO, "My info message"); + + @Test + public void server_dispatches_log_messages_from_log_request() { + List<LogMessage> messages = List.of(MESSAGE_1, MESSAGE_2); + LogDispatcher logDispatcher = mock(LogDispatcher.class); + try (RpcServer server = new RpcServer(0)) { + server.addMethod(new ArchiveLogMessagesMethod(logDispatcher).methodDefinition()); + server.start(); + try (TestClient client = new TestClient(server.listenPort())) { + client.logMessages(messages); + } + } + verify(logDispatcher).handle(new ArrayList<>(messages)); + } + + private static class TestClient implements AutoCloseable { + + private final Supervisor supervisor; + private final Target target; + + TestClient(int logserverPort) { + this.supervisor = new Supervisor(new Transport()); + this.target = supervisor.connectSync(new Spec(logserverPort)); + } + + void logMessages(List<LogMessage> messages) { + byte[] requestPayload = ProtobufSerialization.toLogRequest(messages); + Request request = new Request(ArchiveLogMessagesMethod.METHOD_NAME); + request.parameters().add(new Int8Value((byte)0)); + request.parameters().add(new Int32Value(requestPayload.length)); + request.parameters().add(new DataValue(requestPayload)); + target.invokeSync(request, 10/*seconds*/); + Values returnValues = request.returnValues(); + assertEquals(3, returnValues.size()); + assertEquals(0, returnValues.get(0).asInt8()); + byte[] responsePayload = returnValues.get(2).asData(); + assertEquals(responsePayload.length, returnValues.get(1).asInt32()); + fromLogResponse(responsePayload); // 'void' return type as current response message contains no data + } + + @Override + public void close() { + target.close(); + supervisor.transport().shutdown().join(); + } + } + +}
\ No newline at end of file diff --git a/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java index a97eaf69f84..e3793ee3057 100644 --- a/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java @@ -5,12 +5,13 @@ import com.yahoo.log.LogSetup; import com.yahoo.logserver.handlers.LogHandler; import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin; import com.yahoo.logserver.test.LogDispatcherTestCase; +import org.junit.Test; import java.io.IOException; -import org.junit.*; - -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Unit tests for the Server class. @@ -23,7 +24,7 @@ public class ServerTestCase { public void testStartupAndRegHandlers() throws IOException, InterruptedException { Server.help(); Server server = Server.getInstance(); - server.initialize(18322); + server.initialize(18322, 18323); // TODO Stop using hardcoded ports LogSetup.clearHandlers(); Thread serverThread = new Thread(server); serverThread.start(); diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java deleted file mode 100644 index ece21fbeca7..00000000000 --- a/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * $Id$ - * - */ -package com.yahoo.logserver.formatter.test; - -import com.yahoo.logserver.formatter.LogFormatter; -import com.yahoo.logserver.formatter.LogFormatterManager; -import com.yahoo.logserver.formatter.NullFormatter; -import com.yahoo.logserver.formatter.TextFormatter; - -import org.junit.*; - -import static org.junit.Assert.*; - -/** - * Test the LogFormatterManager - * - * @author Bjorn Borud - */ -public class LogFormatterManagerTestCase { - - /** - * Ensure the system formatters are present - */ - @Test - public void testSystemFormatters() { - LogFormatter lf = LogFormatterManager.getLogFormatter("system.textformatter"); - assertNotNull(lf); - assertEquals(TextFormatter.class, lf.getClass()); - - lf = LogFormatterManager.getLogFormatter("system.nullformatter"); - assertNotNull(lf); - assertEquals(NullFormatter.class, lf.getClass()); - } -} diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java deleted file mode 100644 index a2582e80754..00000000000 --- a/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * $Id$ - * - */ -package com.yahoo.logserver.formatter.test; - -import com.yahoo.log.LogMessage; -import com.yahoo.logserver.formatter.NullFormatter; -import com.yahoo.logserver.test.MockLogEntries; - -import org.junit.*; - -import static org.junit.Assert.*; - -/** - * Test the NullFormatter - * - * @author Bjorn Borud - */ -public class NullFormatterTestCase { - - @Test - public void testNullFormatter() { - NullFormatter nf = new NullFormatter(); - LogMessage[] ms = MockLogEntries.getMessages(); - for (LogMessage m : ms) { - assertEquals(m.toString(), nf.format(m)); - } - } -} diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java deleted file mode 100644 index aee3932fa06..00000000000 --- a/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * $Id$ - * - */ -package com.yahoo.logserver.formatter.test; - -import com.yahoo.log.InvalidLogFormatException; -import com.yahoo.log.LogMessage; -import com.yahoo.logserver.formatter.TextFormatter; -import com.yahoo.logserver.test.MockLogEntries; - -import org.junit.*; - -import static org.junit.Assert.*; - -/** - * Test the TextFormatter - * - * @author Bjorn Borud - */ -public class TextFormatterTestCase { - - /** - * Just simple test to make sure it doesn't die on us - */ - @Test - public void testTextFormatter() { - TextFormatter tf = new TextFormatter(); - LogMessage[] ms = MockLogEntries.getMessages(); - for (int i = 0; i < ms.length; i++) { - System.out.println(tf.format(ms[i])); - } - } - - /** - * Test that a specific log message is formatted correctly - */ - @Test - public void testSpecificMessage() throws InvalidLogFormatException { - String l = "1115200798.195568\texample.yahoo.com\t65819\ttopleveldispatch\tfdispatch.queryperf\tevent\tvalue/1 name=\"query_eval_time_avg_s\" value=0.0229635972697721825"; - String result = "2005-05-04 09:59:58 example.yahoo.com 65819 topleveldispatch fdispatch.queryperf EVENT value/1 name=\"query_eval_time_avg_s\" value=0.0229635972697721825\n"; - LogMessage m = LogMessage.parseNativeFormat(l); - TextFormatter tf = new TextFormatter(); - assertEquals(result, tf.format(m)); - } -} diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java index e4904728d9c..525d02c4298 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java @@ -1,6 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.handlers.archive; +import com.yahoo.io.IOUtils; +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogMessage; +import com.yahoo.plugin.SystemPropertyConfig; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -10,15 +18,9 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import com.yahoo.io.IOUtils; -import com.yahoo.log.InvalidLogFormatException; -import com.yahoo.log.LogMessage; -import com.yahoo.plugin.SystemPropertyConfig; - -import org.junit.*; -import org.junit.rules.TemporaryFolder; - -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author Bjorn Borud @@ -85,8 +87,8 @@ public class ArchiverHandlerTestCase { try { ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), 1024); - LogMessage msg1 = LogMessage.parseNativeFormat("1139322725\thost\tthread\tservice\tcomponent\tinfo\tpayload"); - LogMessage msg2 = LogMessage.parseNativeFormat("1161172200\thost\tthread\tservice\tcomponent\tinfo\tpayload"); + LogMessage msg1 = LogMessage.parseNativeFormat("1139322725\thost\t1/1\tservice\tcomponent\tinfo\tpayload"); + LogMessage msg2 = LogMessage.parseNativeFormat("1161172200\thost\t1/1\tservice\tcomponent\tinfo\tpayload"); assertEquals(tmpDir.getAbsolutePath() + "/2006/02/07/14", a.getPrefix(msg1)); assertEquals(tmpDir.getAbsolutePath() + "/2006/10/18/11", a.getPrefix(msg2)); assertEquals(a.getPrefix(msg1).length(), a.getPrefix(msg2).length()); diff --git a/vespalog/abi-spec.json b/vespalog/abi-spec.json index c5b64b0a66b..6f9f631c6a5 100644 --- a/vespalog/abi-spec.json +++ b/vespalog/abi-spec.json @@ -118,9 +118,13 @@ "public" ], "methods": [ + "public static com.yahoo.log.LogMessage of(java.time.Instant, java.lang.String, long, long, java.lang.String, java.lang.String, java.util.logging.Level, java.lang.String)", + "public java.time.Instant getTimestamp()", "public long getTime()", "public long getTimeInSeconds()", "public java.lang.String getHost()", + "public long getProcessId()", + "public java.util.OptionalLong getThreadId()", "public java.lang.String getThreadProcess()", "public java.lang.String getService()", "public java.lang.String getComponent()", @@ -128,7 +132,9 @@ "public java.lang.String getPayload()", "public static com.yahoo.log.LogMessage parseNativeFormat(java.lang.String)", "public com.yahoo.log.event.Event getEvent()", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" ], "fields": [] }, diff --git a/vespalog/src/main/java/com/yahoo/log/LogMessage.java b/vespalog/src/main/java/com/yahoo/log/LogMessage.java index 1e2b9dfdab0..ac5b4fcfa0e 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogMessage.java +++ b/vespalog/src/main/java/com/yahoo/log/LogMessage.java @@ -1,14 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; +import com.yahoo.log.event.Event; +import com.yahoo.log.event.MalformedEventException; + +import java.time.Instant; +import java.util.Objects; +import java.util.OptionalLong; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.yahoo.log.event.Event; -import com.yahoo.log.event.MalformedEventException; - /** * This class implements the common ground log message used by * the logserver. A LogMessage is immutable. Note that we have @@ -16,6 +19,7 @@ import com.yahoo.log.event.MalformedEventException; * which is used in java.util.logging. * * @author Bjorn Borud + * @author bjorncs */ public class LogMessage { @@ -31,10 +35,10 @@ public class LogMessage "(.+)$" // payload ); - private long time; - private String timeStr; + private Instant time; private String host; - private String threadProcess; + private long processId; + private long threadId; private String service; private String component; private Level level; @@ -45,24 +49,45 @@ public class LogMessage * Private constructor. Log messages should never be instantiated * directly; only as the result of a static factory method. */ - private LogMessage (String timeStr, Long time, String host, String threadProcess, + private LogMessage (Instant time, String host, long processId, long threadId, String service, String component, Level level, String payload) { - this.timeStr = timeStr; this.time = time; this.host = host; - this.threadProcess = threadProcess; + this.processId = processId; + this.threadId = threadId; this.service = service; this.component = component; this.level = level; this.payload = payload; } - public long getTime () {return time;} - public long getTimeInSeconds () {return time / 1000;} + public static LogMessage of( + Instant time, String host, long processId, long threadId, + String service, String component, Level level, String payload) { + return new LogMessage(time, host, processId, threadId, service, component, level, payload); + } + + public Instant getTimestamp() {return time;} + /** + * @deprecated Use {@link #getTimestamp()} + */ + @Deprecated(since = "7", forRemoval = true) + public long getTime () {return time.toEpochMilli();} + /** + * @deprecated Use {@link #getTimestamp()} + */ + @Deprecated(since = "7", forRemoval = true) + public long getTimeInSeconds () {return time.getEpochSecond();} public String getHost () {return host;} - public String getThreadProcess () {return threadProcess;} + public long getProcessId() {return processId;} + public OptionalLong getThreadId() {return threadId > 0 ? OptionalLong.of(threadId) : OptionalLong.empty();} + /** + * @deprecated Use {@link #getProcessId()} / {@link #getThreadId()} + */ + @Deprecated(since = "7", forRemoval = true) + public String getThreadProcess () {return VespaFormat.formatThreadProcess(processId, threadId);} public String getService () {return service;} public String getComponent () {return component;} public Level getLevel () {return level;} @@ -85,19 +110,37 @@ public class LogMessage } Level msgLevel = LogLevel.parse(m.group(6)); - Long timestamp = parseTimestamp(m.group(1)); + Instant timestamp = parseTimestamp(m.group(1)); + String threadProcess = m.group(3); - return new LogMessage(m.group(1), timestamp, m.group(2), m.group(3), + return new LogMessage(timestamp, m.group(2), parseProcessId(threadProcess), parseThreadId(threadProcess), m.group(4), m.group(5), msgLevel, m.group(7)); } - private static long parseTimestamp(String timeStr) throws InvalidLogFormatException { + private static Instant parseTimestamp(String timeStr) throws InvalidLogFormatException { try { - return (long) (Double.parseDouble(timeStr) * 1000); + long nanoseconds = (long) (Double.parseDouble(timeStr) * 1_000_000_000L); + return Instant.ofEpochSecond(0, nanoseconds); } catch (NumberFormatException e) { - throw new InvalidLogFormatException("Invalid time string:" + timeStr); + throw new InvalidLogFormatException("Invalid time string: " + timeStr); + } + } + + private static long parseProcessId(String threadProcess) { + int slashIndex = threadProcess.indexOf('/'); + if (slashIndex == -1) { + return Long.parseLong(threadProcess); + } + return Long.parseLong(threadProcess.substring(0, slashIndex)); + } + + private static long parseThreadId(String threadProcess) { + int slashIndex = threadProcess.indexOf('/'); + if (slashIndex == -1) { + return 0; } + return Long.parseLong(threadProcess.substring(slashIndex + 1)); } /** @@ -117,7 +160,7 @@ public class LogMessage if ((level == LogLevel.EVENT) && (event == null)) { try { event = Event.parse(getPayload()); - event.setTime(time); + event.setTime(time.toEpochMilli()); } catch (MalformedEventException e) { log.log(LogLevel.DEBUG, "Got malformed event: " + getPayload()); @@ -131,6 +174,8 @@ public class LogMessage * Return valid representation of log message. */ public String toString () { + String threadProcess = VespaFormat.formatThreadProcess(processId, threadId); + String timeStr = VespaFormat.formatTime(time); return new StringBuilder(timeStr.length() + host.length() + threadProcess.length() @@ -148,4 +193,25 @@ public class LogMessage .append(payload).append("\n") .toString(); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LogMessage that = (LogMessage) o; + return processId == that.processId && + threadId == that.threadId && + Objects.equals(time, that.time) && + Objects.equals(host, that.host) && + Objects.equals(service, that.service) && + Objects.equals(component, that.component) && + Objects.equals(level, that.level) && + Objects.equals(payload, that.payload) && + Objects.equals(event, that.event); + } + + @Override + public int hashCode() { + return Objects.hash(time, host, processId, threadId, service, component, level, payload, event); + } } diff --git a/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java b/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java index 15aeb347a9b..469f808546f 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java +++ b/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java @@ -39,7 +39,7 @@ public class LogMessageTimeComparator implements Comparator<LogMessage>, Seriali public int compare(LogMessage message1, LogMessage message2) { return ascending ? - Long.valueOf(message1.getTime()).compareTo(Long.valueOf(message2.getTime())) - : Long.valueOf(message2.getTime()).compareTo(Long.valueOf(message1.getTime())); + message1.getTimestamp().compareTo(message2.getTimestamp()) + : message2.getTimestamp().compareTo(message1.getTimestamp()); } } diff --git a/vespalog/src/main/java/com/yahoo/log/VespaFormat.java b/vespalog/src/main/java/com/yahoo/log/VespaFormat.java index c2d5d07e9e1..3d1a2ae0dc8 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaFormat.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaFormat.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; +import java.time.Instant; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -105,6 +106,12 @@ public class VespaFormat { sbuffer.append(timeString.substring(len - 3)); } + static String formatTime(Instant instant) { + StringBuilder builder = new StringBuilder(); + VespaFormat.formatTime(instant.toEpochMilli(), builder); + return builder.toString(); + } + public static String format(String levelName, String component, String componentPrefix, @@ -193,4 +200,11 @@ public class VespaFormat { sbuf.append(" nesting=").append(depth); } + static String formatThreadProcess(long processId, long threadId) { + if (threadId == 0) { + return Long.toString(processId); + } + return processId + "/" + threadId; + } + } |