From bfdd40f8fc37a62851ce52dfc7962d551d3c5f23 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Wed, 27 Mar 2019 17:41:39 +0100 Subject: Remove legacy protocol from logserver --- .../src/main/java/com/yahoo/logserver/Server.java | 43 +---- .../com/yahoo/logserver/net/LogConnection.java | 177 ----------------- .../yahoo/logserver/net/LogConnectionFactory.java | 32 ---- logserver/src/test/files/logEntries.txt | 3 - .../java/com/yahoo/logserver/ServerTestCase.java | 6 +- .../logserver/net/test/LogConnectionTestCase.java | 210 --------------------- .../com/yahoo/logserver/test/MockLogEntries.java | 53 ------ 7 files changed, 5 insertions(+), 519 deletions(-) delete mode 100644 logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java delete mode 100644 logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java delete mode 100644 logserver/src/test/files/logEntries.txt delete mode 100644 logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java delete mode 100644 logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java (limited to 'logserver/src') diff --git a/logserver/src/main/java/com/yahoo/logserver/Server.java b/logserver/src/main/java/com/yahoo/logserver/Server.java index 68ab8be2fba..cc3ef70e62a 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Server.java +++ b/logserver/src/main/java/com/yahoo/logserver/Server.java @@ -4,19 +4,14 @@ package com.yahoo.logserver; import ai.vespa.logserver.protocol.ArchiveLogMessagesMethod; import ai.vespa.logserver.protocol.RpcServer; import com.yahoo.io.FatalErrorHandler; -import com.yahoo.io.Listener; -import com.yahoo.log.LogLevel; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; import com.yahoo.logserver.handlers.HandlerThread; import com.yahoo.logserver.handlers.LogHandler; -import com.yahoo.logserver.net.LogConnectionFactory; import com.yahoo.yolean.system.CatchSignals; -import java.io.IOException; import java.util.HashMap; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Level; import java.util.logging.Logger; /** @@ -42,12 +37,7 @@ public class Server implements Runnable { } private static final int DEFAULT_RPC_LISTEN_PORT = 19080; - // the port is a String because we want to use it as the default - // value of a System.getProperty(). - private static final String LISTEN_PORT = "19081"; - private int listenPort; - private Listener listener; private final LogDispatcher dispatch; private RpcServer rpcServer; @@ -109,25 +99,15 @@ public class Server implements Runnable { /** * Initialize the server and start up all its plugins, - * - * @param listenPort The port on which the logserver accepts log - * messages. - * @param rpcListenPort */ - public void initialize(int listenPort, int rpcListenPort) { + public void initialize(int rpcListenPort) { if (isInitialized) { throw new IllegalStateException(APPNAME + " already initialized"); } - this.listenPort = listenPort; - // plugins registerPluginLoader(new BuiltinPluginLoader()); - // main listener - listener = new Listener(APPNAME); - listener.addSelectLoopPostHook(dispatch); - listener.setFatalErrorHandler(fatalErrorHandler); rpcServer = new RpcServer(rpcListenPort); rpcServer.addMethod(new ArchiveLogMessagesMethod(dispatch).methodDefinition()); } @@ -136,26 +116,11 @@ public class Server implements Runnable { * Sets up the listen port and starts the Listener. Then waits for * Listener to exit. */ + @Override public void run() { - try { - listener.listen(new LogConnectionFactory(dispatch), listenPort); - log.log(LogLevel.CONFIG, APPNAME + ".listenport=" + listenPort); - } catch (IOException e) { - log.log(LogLevel.ERROR, "Unable to initialize", e); - return; - } - - log.fine("Starting listener..."); - listener.start(); log.fine("Starting rpc server..."); rpcServer.start(); Event.started(APPNAME); - try { - listener.join(); - log.fine("listener thread exited"); - } catch (InterruptedException e) { - log.log(Level.WARNING, "Server was interrupted", e); - } } private void setupSignalHandler() { @@ -187,7 +152,6 @@ public class Server implements Runnable { System.out.println(); System.out.println("System properties:"); System.out.println(" - " + APPNAME + ".rpcListenPort (" + DEFAULT_RPC_LISTEN_PORT + ")"); - System.out.println(" - " + APPNAME + ".listenport (" + LISTEN_PORT + ")"); System.out.println(" - " + APPNAME + ".queue.size (" + HandlerThread.DEFAULT_QUEUESIZE + ")"); System.out.println(); } @@ -198,11 +162,10 @@ public class Server implements Runnable { System.exit(0); } - String portString = System.getProperty(APPNAME + ".listenport", LISTEN_PORT); int rpcPort = Integer.parseInt(System.getProperty(APPNAME + ".rpcListenPort", Integer.toString(DEFAULT_RPC_LISTEN_PORT))); Server server = Server.getInstance(); server.setupSignalHandler(); - server.initialize(Integer.parseInt(portString), rpcPort); + server.initialize(rpcPort); Thread t = new Thread(server, "logserver main"); t.start(); diff --git a/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java b/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java deleted file mode 100644 index cfe4e425ff3..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.net; - -import com.yahoo.io.Connection; -import com.yahoo.io.ReadLine; -import com.yahoo.log.InvalidLogFormatException; -import com.yahoo.log.LogLevel; -import com.yahoo.log.LogMessage; -import com.yahoo.logserver.LogDispatcher; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.SelectionKey; -import java.nio.channels.SocketChannel; -import java.util.HashSet; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * TODO - * - * - * @author Bjorn Borud - */ - -public class LogConnection implements Connection { - private static final Logger log = Logger.getLogger(LogConnection.class.getName()); - - public static final int READBUFFER_SIZE = (32 * 1024); - - // the set of active connections - private static final Set activeConnections = new HashSet<>(); - - private final SocketChannel socket; - private final LogDispatcher dispatcher; - - private final ByteBuffer readBuffer = ByteBuffer.allocateDirect(READBUFFER_SIZE); - - // counters - private long totalBytesRead = 0; - - public LogConnection(SocketChannel socket, LogDispatcher dispatcher) { - this.socket = socket; - this.dispatcher = dispatcher; - - addToActiveSet(this); - - } - - /** - * Return a shallow copy of the set of active connections. - * - */ - public static Set getActiveConnections () { - synchronized(activeConnections) { - return new HashSet<>(activeConnections); - } - } - - /** - * @return Return total number of bytes read from connection - */ - public long getTotalBytesRead () { - return totalBytesRead; - } - - /** - * Internal method for adding connection to the set - * of active connections. - * - * @param connection The connection to be added - */ - private static void addToActiveSet (LogConnection connection) { - synchronized(activeConnections) { - activeConnections.add(connection); - } - } - - /** - * Internal method to remove connection from the set of - * active connections. - * - * @param connection The connection to remove - * @throws IllegalStateException if the connection does not - * exist in the set - * - */ - private static void removeFromActiveSet (LogConnection connection) { - synchronized(activeConnections) { - activeConnections.remove(connection); - } - } - - public void connect () throws IOException { - throw new RuntimeException("connect() is not supposed to be called"); - } - - public void write () { - throw new UnsupportedOperationException(); - } - - public void read() throws IOException { - if (! readBuffer.hasRemaining()) { - - try { - readBuffer.putChar(readBuffer.capacity() - 2, '\n'); - readBuffer.flip(); - String s = ReadLine.readLine(readBuffer); - if (s == null) { - return; - } - log.log(LogLevel.DEBUG, "Log message too long. Message from " - + socket.socket().getInetAddress() + " exceeds " - + readBuffer.capacity() + ". The message was: " + s); - - LogMessage msg = LogMessage.parseNativeFormat(s); - dispatcher.handle(msg); - } - catch (InvalidLogFormatException e) { - log.log(LogLevel.DEBUG, "Invalid log message", e); - } - finally { - readBuffer.clear(); - } - return; - } - - int ret = socket.read(readBuffer); - if (ret == -1) { - close(); - return; - } - - if (ret == 0) { - if (log.isLoggable(Level.FINE)) { - log.log(LogLevel.DEBUG, "zero byte read occurred"); - } - } - - // update global counter - totalBytesRead += ret; - - readBuffer.flip(); - - String s; - while ((s = ReadLine.readLine(readBuffer)) != null) { - try { - LogMessage msg = LogMessage.parseNativeFormat(s); - dispatcher.handle(msg); - } - catch (InvalidLogFormatException e) { - log.log(LogLevel.DEBUG, "Invalid log message", e); - } - } - } - - public void close() throws IOException { - if (log.isLoggable(Level.FINE)) { - log.log(LogLevel.INFO, this + ": closing"); - } - socket.close(); - removeFromActiveSet(this); - } - - public int selectOps() { - return SelectionKey.OP_READ; - } - - public SocketChannel socketChannel() { - return socket; - } - -} diff --git a/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java b/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java deleted file mode 100644 index c41484f3814..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.net; - -import com.yahoo.io.Connection; -import com.yahoo.io.ConnectionFactory; -import com.yahoo.io.Listener; -import com.yahoo.logserver.LogDispatcher; - -import java.nio.channels.SocketChannel; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author Bjorn Borud - */ -public class LogConnectionFactory implements ConnectionFactory { - private static final Logger log = Logger.getLogger(LogConnectionFactory.class.getName()); - - private final LogDispatcher dispatcher; - - public LogConnectionFactory(LogDispatcher dispatcher) { - this.dispatcher = dispatcher; - } - - public Connection newConnection(SocketChannel socket, Listener listener) { - if (log.isLoggable(Level.FINE)) { - log.fine("New connection: " + socket); - } - return new LogConnection(socket, - dispatcher); - } -} diff --git a/logserver/src/test/files/logEntries.txt b/logserver/src/test/files/logEntries.txt deleted file mode 100644 index 776e3de882d..00000000000 --- a/logserver/src/test/files/logEntries.txt +++ /dev/null @@ -1,3 +0,0 @@ -1096639280.524133 malfunction 26851 - logtest info Starting up, called as ./log/logtest -1096639280.524164 malfunction 26851 - logtest info backslash: \\ -1096639280.524133 malfunction 26851 - logtest fatal Starting up, called as ./log/logtest diff --git a/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java index e3793ee3057..23fa0b868b3 100644 --- a/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java @@ -7,8 +7,6 @@ import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin; import com.yahoo.logserver.test.LogDispatcherTestCase; import org.junit.Test; -import java.io.IOException; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -21,10 +19,10 @@ import static org.junit.Assert.fail; public class ServerTestCase { @Test - public void testStartupAndRegHandlers() throws IOException, InterruptedException { + public void testStartupAndRegHandlers() { Server.help(); Server server = Server.getInstance(); - server.initialize(18322, 18323); // TODO Stop using hardcoded ports + server.initialize(0); LogSetup.clearHandlers(); Thread serverThread = new Thread(server); serverThread.start(); diff --git a/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java b/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java deleted file mode 100644 index a68b62b122b..00000000000 --- a/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.net.test; - -import com.yahoo.log.LogLevel; -import com.yahoo.log.LogMessage; -import com.yahoo.logserver.LogDispatcher; -import com.yahoo.logserver.handlers.AbstractLogHandler; -import com.yahoo.logserver.net.LogConnection; -import com.yahoo.logserver.test.MockLogEntries; -import org.junit.Test; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.ServerSocketChannel; -import java.nio.channels.SocketChannel; -import java.nio.charset.Charset; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.BrokenBarrierException; -import java.util.concurrent.CyclicBarrier; -import java.util.logging.Logger; - -import static org.junit.Assert.*; - -/** - * Unit tests for the log connection class. Wow is this too - * complex! - * - * @author Bjorn Borud - */ -public class LogConnectionTestCase { - private static final Logger log = Logger.getLogger(LogConnectionTestCase.class.getName()); - - private static final Charset charset = Charset.forName("utf-8"); - private static final ByteBuffer bigBuffer; - private int port; - - static { - StringBuilder sb = new StringBuilder(LogConnection.READBUFFER_SIZE * 6) - .append(MockLogEntries.getMessages()[0].toString()) - .append(MockLogEntries.getMessages()[1].toString()) - .append(MockLogEntries.getMessages()[2].toString()); - - // get a valid log message prefix - String prefix = MockLogEntries.getMessages()[2].toString(); - prefix = prefix.substring(0, prefix.length() - 1); - sb.append(prefix); - - // fill up the remaining buffer with rubbish to make - // it too long - for (int i = 0; i < (LogConnection.READBUFFER_SIZE * 3); i++) { - sb.append("a"); - } - sb.append('\n'); - sb.append(MockLogEntries.getMessages()[2].toString()); - bigBuffer = charset.encode(sb.toString()); - } - - /** - * this code is nothing short of completely hideous. the exception - * handling is awful and the code is messy, but it should be a fairly - * efficient and robust way of testing buffer overflow conditions. - */ - @Test - public void testOverflow() { - final CyclicBarrier barrier = new CyclicBarrier(2); - - - // this inner class is used to define a mock handler - // which will help us examine the messages actually - // handled. - class MockHandler extends AbstractLogHandler { - private final List messages = new LinkedList(); - - public boolean doHandle(LogMessage msg) { - messages.add(msg); - return true; - } - - public List getMessages() { - return messages; - } - - public void flush() { - } - - public void close() { - } - - public String toString() { - return null; - } - } - - Thread serverThread = new Thread() { - public void run() { - ServerSocketChannel ss = setUpListenSocket(); - if (ss == null) { - fail("unable to set up listen socket"); - return; - } - - setPort(ss.socket().getLocalPort()); - - // listen port is up now so we can trigger the barrier - try { - barrier.await(); - - while (!Thread.currentThread().isInterrupted()) { - SocketChannel s = ss.accept(); - pushBigBuffer(s); - s.close(); - } - } catch (BrokenBarrierException e) { - fail(e.getMessage()); - return; - } catch (InterruptedException | java.nio.channels.ClosedByInterruptException e) { - return; - } catch (IOException e) { - log.log(LogLevel.ERROR, "argh", e); - fail(); - return; - } - - } - }; - serverThread.start(); - assertTrue(serverThread.isAlive()); - - try { - barrier.await(); - } catch (BrokenBarrierException e) { - fail(e.getMessage()); - return; - } catch (InterruptedException e) { - return; - } - - SocketChannel sock; - try { - sock = SocketChannel.open(new InetSocketAddress("localhost", port)); - } catch (IOException e) { - fail(e.getMessage()); - return; - } - - LogDispatcher dispatcher = new LogDispatcher(); - MockHandler mock = new MockHandler(); - assertTrue(mock.getName().endsWith("MockHandler")); - dispatcher.registerLogHandler(mock); - LogConnection logConnection = - new LogConnection(sock, dispatcher); - - try { - for (int i = 0; i < 100; i++) { - logConnection.read(); - } - } catch (java.nio.channels.ClosedChannelException e) { - // ignore, this is normal - } catch (IOException e) { - log.log(LogLevel.ERROR, "error during reading", e); - } - - // there should be 5 messages - assertEquals(5, mock.getMessages().size()); - assertEquals(5, mock.getCount()); - - // the 4'th message should be long - String m = (mock.getMessages().get(3)).getPayload(); - assertTrue(m.length() > 10000); - - serverThread.interrupt(); - try { - serverThread.join(); - assertTrue(true); - } catch (InterruptedException e) { - fail(); - } - } - - private void pushBigBuffer(SocketChannel socket) { - try { - ByteBuffer slice = bigBuffer.slice(); - while (slice.hasRemaining()) { - @SuppressWarnings("unused") - int ret = socket.write(slice); - } - } catch (java.io.IOException e) { - - } - } - - private void setPort(int port) { - this.port = port; - } - - private ServerSocketChannel setUpListenSocket() { - int p = 18327; - try { - ServerSocketChannel s = ServerSocketChannel.open(); - s.socket().setReuseAddress(true); - s.socket().bind(new InetSocketAddress("127.0.0.1", p)); - return s; - } catch (Exception e) { - fail(e.getMessage()); - } - return null; - } -} diff --git a/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java b/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java deleted file mode 100644 index 67a51843de9..00000000000 --- a/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.test; - -import com.yahoo.log.InvalidLogFormatException; -import com.yahoo.log.LogMessage; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; - -/** - * This class is used to gain access to a bunch of log entries - * so we can use the same log messages in several different tests - * - * @author Bjorn Borud - */ -public class MockLogEntries { - private static final MockLogEntries instance = new MockLogEntries(); - - private final LogMessage[] messages; - - /** - * Private constructor which reads the log messages and builds - * an array of LogMessage entries. - */ - private MockLogEntries() { - List msgs = new LinkedList(); - try { - String name = "src/test/files/logEntries.txt"; - BufferedReader br = new BufferedReader(new FileReader(name)); - for (String line = br.readLine(); line != null; line = br.readLine()) { - LogMessage m = LogMessage.parseNativeFormat(line); - msgs.add(m); - } - } catch (InvalidLogFormatException | IOException e) { - // do nothing - } - - LogMessage[] m = new LogMessage[msgs.size()]; - msgs.toArray(m); - messages = m; - } - - /** - * Return the LogMessage instances we've constructed from the - * stored log messages we have. - */ - public static LogMessage[] getMessages() { - return instance.messages; - } -} -- cgit v1.2.3