Merge pull request #9172 from vespa-engine/bjorncs/logserver-cleanup

Remove legacy protocol from logserver

7 files changed, 5 insertions, 519 deletions

diff --git a/logserver/src/main/java/com/yahoo/logserver/Server.java b/logserver/src/main/java/com/yahoo/logserver/Server.java
index 68ab8be2fba..cc3ef70e62a 100644
--- a/logserver/src/main/java/com/yahoo/logserver/Server.java
+++ b/logserver/src/main/java/com/yahoo/logserver/Server.java
@@ -4,19 +4,14 @@ package com.yahoo.logserver;
 import ai.vespa.logserver.protocol.ArchiveLogMessagesMethod;
 import ai.vespa.logserver.protocol.RpcServer;
 import com.yahoo.io.FatalErrorHandler;
-import com.yahoo.io.Listener;
-import com.yahoo.log.LogLevel;
 import com.yahoo.log.LogSetup;
 import com.yahoo.log.event.Event;
 import com.yahoo.logserver.handlers.HandlerThread;
 import com.yahoo.logserver.handlers.LogHandler;
-import com.yahoo.logserver.net.LogConnectionFactory;
 import com.yahoo.yolean.system.CatchSignals;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
@@ -42,12 +37,7 @@ public class Server implements Runnable {
     }
 
     private static final int DEFAULT_RPC_LISTEN_PORT = 19080;
-    // the port is a String because we want to use it as the default
-    // value of a System.getProperty().
-    private static final String LISTEN_PORT = "19081";
 
-    private int listenPort;
-    private Listener listener;
     private final LogDispatcher dispatch;
     private RpcServer rpcServer;
 
@@ -109,25 +99,15 @@ public class Server implements Runnable {
 
     /**
      * Initialize the server and start up all its plugins,
-     *
-     * @param listenPort The port on which the logserver accepts log
-     *                   messages.
-     * @param rpcListenPort
      */
-    public void initialize(int listenPort, int rpcListenPort) {
+    public void initialize(int rpcListenPort) {
         if (isInitialized) {
             throw new IllegalStateException(APPNAME + " already initialized");
         }
 
-        this.listenPort = listenPort;
-
         // plugins
         registerPluginLoader(new BuiltinPluginLoader());
 
-        // main listener
-        listener = new Listener(APPNAME);
-        listener.addSelectLoopPostHook(dispatch);
-        listener.setFatalErrorHandler(fatalErrorHandler);
         rpcServer = new RpcServer(rpcListenPort);
         rpcServer.addMethod(new ArchiveLogMessagesMethod(dispatch).methodDefinition());
     }
@@ -136,26 +116,11 @@ public class Server implements Runnable {
      * Sets up the listen port and starts the Listener.  Then waits for
      * Listener to exit.
      */
+    @Override
     public void run() {
-        try {
-            listener.listen(new LogConnectionFactory(dispatch), listenPort);
-            log.log(LogLevel.CONFIG, APPNAME + ".listenport=" + listenPort);
-        } catch (IOException e) {
-            log.log(LogLevel.ERROR, "Unable to initialize", e);
-            return;
-        }
-
-        log.fine("Starting listener...");
-        listener.start();
         log.fine("Starting rpc server...");
         rpcServer.start();
         Event.started(APPNAME);
-        try {
-            listener.join();
-            log.fine("listener thread exited");
-        } catch (InterruptedException e) {
-            log.log(Level.WARNING, "Server was interrupted", e);
-        }
     }
 
     private void setupSignalHandler() {
@@ -187,7 +152,6 @@ public class Server implements Runnable {
         System.out.println();
         System.out.println("System properties:");
         System.out.println(" - " + APPNAME + ".rpcListenPort (" + DEFAULT_RPC_LISTEN_PORT + ")");
-        System.out.println(" - " + APPNAME + ".listenport (" + LISTEN_PORT + ")");
         System.out.println(" - " + APPNAME + ".queue.size (" + HandlerThread.DEFAULT_QUEUESIZE + ")");
         System.out.println();
     }
@@ -198,11 +162,10 @@ public class Server implements Runnable {
             System.exit(0);
         }
 
-        String portString = System.getProperty(APPNAME + ".listenport", LISTEN_PORT);
         int rpcPort = Integer.parseInt(System.getProperty(APPNAME + ".rpcListenPort", Integer.toString(DEFAULT_RPC_LISTEN_PORT)));
         Server server = Server.getInstance();
         server.setupSignalHandler();
-        server.initialize(Integer.parseInt(portString), rpcPort);
+        server.initialize(rpcPort);
 
         Thread t = new Thread(server, "logserver main");
         t.start();
diff --git a/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java b/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java
deleted file mode 100644
index cfe4e425ff3..00000000000
--- a/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java
+++ /dev/null
@@ -1,177 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.logserver.net;
-
-import com.yahoo.io.Connection;
-import com.yahoo.io.ReadLine;
-import com.yahoo.log.InvalidLogFormatException;
-import com.yahoo.log.LogLevel;
-import com.yahoo.log.LogMessage;
-import com.yahoo.logserver.LogDispatcher;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.SelectionKey;
-import java.nio.channels.SocketChannel;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * TODO
- * <UL>
- *  <LI> send invalid log messages to somewhere so they can be
- *       analyzed and errors can be corrected.
- * </UL>
- *
- * @author Bjorn Borud
- */
-
-public class LogConnection implements Connection {
-    private static final Logger log = Logger.getLogger(LogConnection.class.getName());
-
-    public static final int READBUFFER_SIZE = (32 * 1024);
-
-    // the set of active connections
-    private static final Set<LogConnection> activeConnections = new HashSet<>();
-
-    private final SocketChannel socket;
-    private final LogDispatcher dispatcher;
-
-    private final ByteBuffer readBuffer = ByteBuffer.allocateDirect(READBUFFER_SIZE);
-
-    // counters
-    private long totalBytesRead = 0;
-
-    public LogConnection(SocketChannel socket, LogDispatcher dispatcher) {
-        this.socket = socket;
-        this.dispatcher = dispatcher;
-
-        addToActiveSet(this);
-
-    }
-
-    /**
-     * Return a shallow copy of the set of active connections.
-     *
-     */
-    public static Set<LogConnection> getActiveConnections () {
-        synchronized(activeConnections) {
-        	return new HashSet<>(activeConnections);
-        }
-    }
-
-    /**
-     * @return Return total number of bytes read from connection
-     */
-    public long getTotalBytesRead () {
-        return totalBytesRead;
-    }
-
-    /**
-     * Internal method for adding connection to the set
-     * of active connections.
-     *
-     * @param connection The connection to be added
-     */
-    private static void addToActiveSet (LogConnection connection) {
-        synchronized(activeConnections) {
-            activeConnections.add(connection);
-        }
-    }
-
-    /**
-     * Internal method to remove connection from the set of
-     * active connections.
-     *
-     * @param connection The connection to remove
-     * @throws IllegalStateException if the connection does not
-     *                               exist in the set
-     *
-     */
-    private static void removeFromActiveSet (LogConnection connection) {
-        synchronized(activeConnections) {
-            activeConnections.remove(connection);
-        }
-    }
-
-    public void connect () throws IOException {
-        throw new RuntimeException("connect() is not supposed to be called");
-    }
-
-    public void write () {
-        throw new UnsupportedOperationException();
-    }
-
-    public void read() throws IOException {
-        if (! readBuffer.hasRemaining()) {
-
-            try {
-                readBuffer.putChar(readBuffer.capacity() - 2, '\n');
-                readBuffer.flip();
-                String s = ReadLine.readLine(readBuffer);
-                if (s == null) {
-                    return;
-                }
-                log.log(LogLevel.DEBUG, "Log message too long. Message from "
-                        + socket.socket().getInetAddress() +  " exceeds "
-                        + readBuffer.capacity() + ". The message was: " + s);
-
-                LogMessage msg = LogMessage.parseNativeFormat(s);
-                dispatcher.handle(msg);
-            }
-            catch (InvalidLogFormatException e) {
-                log.log(LogLevel.DEBUG, "Invalid log message", e);
-            }
-            finally {
-                readBuffer.clear();
-            }
-            return;
-        }
-
-        int ret = socket.read(readBuffer);
-        if (ret == -1) {
-            close();
-            return;
-        }
-
-        if (ret == 0) {
-            if (log.isLoggable(Level.FINE)) {
-                log.log(LogLevel.DEBUG, "zero byte read occurred");
-             }
-        }
-
-        // update global counter
-        totalBytesRead += ret;
-
-        readBuffer.flip();
-
-        String s;
-        while ((s = ReadLine.readLine(readBuffer)) != null) {
-            try {
-                LogMessage msg = LogMessage.parseNativeFormat(s);
-                dispatcher.handle(msg);
-            }
-            catch (InvalidLogFormatException e) {
-                log.log(LogLevel.DEBUG, "Invalid log message", e);
-            }
-        }
-    }
-
-    public void close() throws IOException {
-        if (log.isLoggable(Level.FINE)) {
-            log.log(LogLevel.INFO, this + ": closing");
-        }
-        socket.close();
-        removeFromActiveSet(this);
-    }
-
-    public int selectOps() {
-        return SelectionKey.OP_READ;
-    }
-
-    public SocketChannel socketChannel() {
-        return socket;
-    }
-
-}
diff --git a/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java b/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java
deleted file mode 100644
index c41484f3814..00000000000
--- a/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java
+++ /dev/null
@@ -1,32 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.logserver.net;
-
-import com.yahoo.io.Connection;
-import com.yahoo.io.ConnectionFactory;
-import com.yahoo.io.Listener;
-import com.yahoo.logserver.LogDispatcher;
-
-import java.nio.channels.SocketChannel;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * @author Bjorn Borud
- */
-public class LogConnectionFactory implements ConnectionFactory {
-    private static final Logger log = Logger.getLogger(LogConnectionFactory.class.getName());
-
-    private final LogDispatcher dispatcher;
-
-    public LogConnectionFactory(LogDispatcher dispatcher) {
-        this.dispatcher = dispatcher;
-    }
-
-    public Connection newConnection(SocketChannel socket, Listener listener) {
-        if (log.isLoggable(Level.FINE)) {
-            log.fine("New connection: " + socket);
-        }
-        return new LogConnection(socket,
-                                 dispatcher);
-    }
-}
diff --git a/logserver/src/test/files/logEntries.txt b/logserver/src/test/files/logEntries.txt
deleted file mode 100644
index 776e3de882d..00000000000
--- a/logserver/src/test/files/logEntries.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-1096639280.524133	malfunction	26851	-	logtest	info	Starting up, called as ./log/logtest
-1096639280.524164	malfunction	26851	-	logtest	info	backslash: \\
-1096639280.524133	malfunction	26851	-	logtest	fatal	Starting up, called as ./log/logtest
diff --git a/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java
index e3793ee3057..23fa0b868b3 100644
--- a/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java
+++ b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java
@@ -7,8 +7,6 @@ import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin;
 import com.yahoo.logserver.test.LogDispatcherTestCase;
 import org.junit.Test;
 
-import java.io.IOException;
-
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -21,10 +19,10 @@ import static org.junit.Assert.fail;
 public class ServerTestCase {
 
     @Test
-    public void testStartupAndRegHandlers() throws IOException, InterruptedException {
+    public void testStartupAndRegHandlers() {
         Server.help();
         Server server = Server.getInstance();
-        server.initialize(18322, 18323); // TODO Stop using hardcoded ports
+        server.initialize(0);
         LogSetup.clearHandlers();
         Thread serverThread = new Thread(server);
         serverThread.start();
diff --git a/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java b/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java
deleted file mode 100644
index a68b62b122b..00000000000
--- a/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java
+++ /dev/null
@@ -1,210 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.logserver.net.test;
-
-import com.yahoo.log.LogLevel;
-import com.yahoo.log.LogMessage;
-import com.yahoo.logserver.LogDispatcher;
-import com.yahoo.logserver.handlers.AbstractLogHandler;
-import com.yahoo.logserver.net.LogConnection;
-import com.yahoo.logserver.test.MockLogEntries;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.nio.channels.ServerSocketChannel;
-import java.nio.channels.SocketChannel;
-import java.nio.charset.Charset;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.CyclicBarrier;
-import java.util.logging.Logger;
-
-import static org.junit.Assert.*;
-
-/**
- * Unit tests for the log connection class.  Wow is this too
- * complex!
- *
- * @author Bjorn Borud
- */
-public class LogConnectionTestCase {
-    private static final Logger log = Logger.getLogger(LogConnectionTestCase.class.getName());
-
-    private static final Charset charset = Charset.forName("utf-8");
-    private static final ByteBuffer bigBuffer;
-    private int port;
-
-    static {
-        StringBuilder sb = new StringBuilder(LogConnection.READBUFFER_SIZE * 6)
-                .append(MockLogEntries.getMessages()[0].toString())
-                .append(MockLogEntries.getMessages()[1].toString())
-                .append(MockLogEntries.getMessages()[2].toString());
-
-        // get a valid log message prefix
-        String prefix = MockLogEntries.getMessages()[2].toString();
-        prefix = prefix.substring(0, prefix.length() - 1);
-        sb.append(prefix);
-
-        // fill up the remaining buffer with rubbish to make
-        // it too long
-        for (int i = 0; i < (LogConnection.READBUFFER_SIZE * 3); i++) {
-            sb.append("a");
-        }
-        sb.append('\n');
-        sb.append(MockLogEntries.getMessages()[2].toString());
-        bigBuffer = charset.encode(sb.toString());
-    }
-
-    /**
-     * this code is nothing short of completely hideous.  the exception
-     * handling is awful and the code is messy, but it should be a fairly
-     * efficient and robust way of testing buffer overflow conditions.
-     */
-    @Test
-    public void testOverflow() {
-        final CyclicBarrier barrier = new CyclicBarrier(2);
-
-
-        // this inner class is used to define a mock handler
-        // which will help us examine the messages actually
-        // handled.
-        class MockHandler extends AbstractLogHandler {
-            private final List<LogMessage> messages = new LinkedList<LogMessage>();
-
-            public boolean doHandle(LogMessage msg) {
-                messages.add(msg);
-                return true;
-            }
-
-            public List<LogMessage> getMessages() {
-                return messages;
-            }
-
-            public void flush() {
-            }
-
-            public void close() {
-            }
-
-            public String toString() {
-                return null;
-            }
-        }
-
-        Thread serverThread = new Thread() {
-            public void run() {
-                ServerSocketChannel ss = setUpListenSocket();
-                if (ss == null) {
-                    fail("unable to set up listen socket");
-                    return;
-                }
-
-                setPort(ss.socket().getLocalPort());
-
-                // listen port is up now so we can trigger the barrier
-                try {
-                    barrier.await();
-
-                    while (!Thread.currentThread().isInterrupted()) {
-                        SocketChannel s = ss.accept();
-                        pushBigBuffer(s);
-                        s.close();
-                    }
-                } catch (BrokenBarrierException e) {
-                    fail(e.getMessage());
-                    return;
-                } catch (InterruptedException | java.nio.channels.ClosedByInterruptException e) {
-                    return;
-                } catch (IOException e) {
-                    log.log(LogLevel.ERROR, "argh", e);
-                    fail();
-                    return;
-                }
-
-            }
-        };
-        serverThread.start();
-        assertTrue(serverThread.isAlive());
-
-        try {
-            barrier.await();
-        } catch (BrokenBarrierException e) {
-            fail(e.getMessage());
-            return;
-        } catch (InterruptedException e) {
-            return;
-        }
-
-        SocketChannel sock;
-        try {
-            sock = SocketChannel.open(new InetSocketAddress("localhost", port));
-        } catch (IOException e) {
-            fail(e.getMessage());
-            return;
-        }
-
-        LogDispatcher dispatcher = new LogDispatcher();
-        MockHandler mock = new MockHandler();
-        assertTrue(mock.getName().endsWith("MockHandler"));
-        dispatcher.registerLogHandler(mock);
-        LogConnection logConnection =
-                new LogConnection(sock, dispatcher);
-
-        try {
-            for (int i = 0; i < 100; i++) {
-                logConnection.read();
-            }
-        } catch (java.nio.channels.ClosedChannelException e) {
-            // ignore, this is normal
-        } catch (IOException e) {
-            log.log(LogLevel.ERROR, "error during reading", e);
-        }
-
-        // there should be 5 messages
-        assertEquals(5, mock.getMessages().size());
-        assertEquals(5, mock.getCount());
-
-        // the 4'th message should be long
-        String m = (mock.getMessages().get(3)).getPayload();
-        assertTrue(m.length() > 10000);
-
-        serverThread.interrupt();
-        try {
-            serverThread.join();
-            assertTrue(true);
-        } catch (InterruptedException e) {
-            fail();
-        }
-    }
-
-    private void pushBigBuffer(SocketChannel socket) {
-        try {
-            ByteBuffer slice = bigBuffer.slice();
-            while (slice.hasRemaining()) {
-                @SuppressWarnings("unused")
-                int ret = socket.write(slice);
-            }
-        } catch (java.io.IOException e) {
-
-        }
-    }
-
-    private void setPort(int port) {
-        this.port = port;
-    }
-
-    private ServerSocketChannel setUpListenSocket() {
-        int p = 18327;
-        try {
-            ServerSocketChannel s = ServerSocketChannel.open();
-            s.socket().setReuseAddress(true);
-            s.socket().bind(new InetSocketAddress("127.0.0.1", p));
-            return s;
-        } catch (Exception e) {
-            fail(e.getMessage());
-        }
-        return null;
-    }
-}
diff --git a/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java b/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java
deleted file mode 100644
index 67a51843de9..00000000000
--- a/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java
+++ /dev/null
@@ -1,53 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.logserver.test;
-
-import com.yahoo.log.InvalidLogFormatException;
-import com.yahoo.log.LogMessage;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * This class is used to gain access to a bunch of log entries
- * so we can use the same log messages in several different tests
- *
- * @author Bjorn Borud
- */
-public class MockLogEntries {
-    private static final MockLogEntries instance = new MockLogEntries();
-
-    private final LogMessage[] messages;
-
-    /**
-     * Private constructor which reads the log messages and builds
-     * an array of LogMessage entries.
-     */
-    private MockLogEntries() {
-        List<LogMessage> msgs = new LinkedList<LogMessage>();
-        try {
-            String name = "src/test/files/logEntries.txt";
-            BufferedReader br = new BufferedReader(new FileReader(name));
-            for (String line = br.readLine(); line != null; line = br.readLine()) {
-                LogMessage m = LogMessage.parseNativeFormat(line);
-                msgs.add(m);
-            }
-        } catch (InvalidLogFormatException | IOException e) {
-            // do nothing
-        }
-
-        LogMessage[] m = new LogMessage[msgs.size()];
-        msgs.toArray(m);
-        messages = m;
-    }
-
-    /**
-     * Return the LogMessage instances we've constructed from the
-     * stored log messages we have.
-     */
-    public static LogMessage[] getMessages() {
-        return instance.messages;
-    }
-}