Revert "Revert "Make it possible to limit number of log lines returned" (#24168)"

This reverts commit d058fe597d6ba98f27c146cf725752aab1405772.

4 files changed, 23 insertions, 36 deletions

diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
index d151eb6862d..72a399744f3 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
@@ -4,21 +4,15 @@ package com.yahoo.container.handler;
 import com.yahoo.component.annotation.Inject;
 import com.yahoo.container.core.LogHandlerConfig;
 import com.yahoo.container.jdisc.AsyncHttpResponse;
-import com.yahoo.container.jdisc.ContentChannelOutputStream;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
 import com.yahoo.jdisc.handler.CompletionHandler;
 import com.yahoo.jdisc.handler.ContentChannel;
 
-import java.io.IOException;
-import java.io.InterruptedIOException;
 import java.io.OutputStream;
-import java.nio.ByteBuffer;
 import java.time.Instant;
 import java.util.Optional;
 import java.util.concurrent.Executor;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Level;
 
 public class LogHandler extends ThreadedHttpRequestHandler {
@@ -42,6 +36,8 @@ public class LogHandler extends ThreadedHttpRequestHandler {
                                .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MIN);
         Instant to = Optional.ofNullable(request.getProperty("to"))
                              .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MAX);
+        long maxLines = Optional.ofNullable(request.getProperty("maxLines"))
+                                .map(Long::valueOf).orElse(100_000L);
         Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname"));
 
         return new AsyncHttpResponse(200) {
@@ -50,7 +46,7 @@ public class LogHandler extends ThreadedHttpRequestHandler {
             @Override
             public void render(OutputStream output, ContentChannel networkChannel, CompletionHandler handler) {
                 try (output) {
-                    logReader.writeLogs(output, from, to, hostname);
+                    logReader.writeLogs(output, from, to, maxLines, hostname);
                 }
                 catch (Throwable t) {
                     log.log(Level.WARNING, "Failed reading logs from " + from + " to " + to, t);
@@ -62,6 +58,4 @@ public class LogHandler extends ThreadedHttpRequestHandler {
         };
     }
 
-
-
 }
diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
index 93881b52eb6..dae4b940307 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
@@ -59,9 +59,10 @@ class LogReader {
         this.logFilePattern = logFilePattern;
     }
 
-    void writeLogs(OutputStream out, Instant from, Instant to, Optional<String> hostname) {
+    void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Optional<String> hostname) {
         double fromSeconds = from.getEpochSecond() + from.getNano() / 1e9;
         double toSeconds = to.getEpochSecond() + to.getNano() / 1e9;
+        long linesWritten = 0;
         BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out));
         for (List<Path> logs : getMatchingFiles(from, to)) {
             List<LogLineIterator> logLineIterators = new ArrayList<>();
@@ -73,6 +74,7 @@ class LogReader {
                 Iterator<LineWithTimestamp> lines = Iterators.mergeSorted(logLineIterators,
                                                                           Comparator.comparingDouble(LineWithTimestamp::timestamp));
                 while (lines.hasNext()) {
+                    if (linesWritten++ >= maxLines) return;
                     String line = lines.next().line();
                     writer.write(line);
                     writer.newLine();
@@ -187,16 +189,7 @@ class LogReader {
 
     }
 
-    private static class LineWithTimestamp {
-        final String line;
-        final double timestamp;
-        LineWithTimestamp(String line, double timestamp) {
-            this.line = line;
-            this.timestamp = timestamp;
-        }
-        String line() { return line; }
-        double timestamp() { return timestamp; }
-    }
+    private record LineWithTimestamp(String line, double timestamp) { }
 
     /** Returns log files which may have relevant entries, grouped and sorted by {@link #extractTimestamp(Path)} — the first and last group must be filtered. */
     private List<List<Path>> getMatchingFiles(Instant from, Instant to) {
diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java
index 3d9a2360e77..8b698b25ab2 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java
@@ -51,7 +51,7 @@ public class LogHandlerTest {
         }
 
         @Override
-        protected void writeLogs(OutputStream out, Instant from, Instant to, Optional<String> hostname)  {
+        protected void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Optional<String> hostname)  {
             try {
                 if (to.isAfter(Instant.ofEpochMilli(1000))) {
                     out.write("newer log".getBytes());
diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
index 106b8cef35f..e98f8cac276 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
@@ -37,7 +37,7 @@ public class LogReaderTest {
 
     @BeforeEach
     public void setup() throws IOException {
-        logDirectory = newFolder(folder, "opt/vespa/logs").toPath();
+        logDirectory = Files.createDirectories(folder.toPath().resolve("opt/vespa/logs"));
         // Log archive paths and file names indicate what hour they contain logs for, with the start of that hour.
         // Multiple entries may exist for each hour.
         Files.createDirectories(logDirectory.resolve("1970/01/01"));
@@ -59,7 +59,7 @@ public class LogReaderTest {
     void testThatLogsOutsideRangeAreExcluded() {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
-        logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), Optional.empty());
+        logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), 100, Optional.empty());
 
         assertEquals(log100 + logv11 + log110, baos.toString(UTF_8));
     }
@@ -68,7 +68,7 @@ public class LogReaderTest {
     void testThatLogsNotMatchingRegexAreExcluded() {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*"));
-        logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty());
+        logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty());
 
         assertEquals(log101 + logv11, baos.toString(UTF_8));
     }
@@ -78,7 +78,7 @@ public class LogReaderTest {
     void testZippedStreaming() {
         ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream();
         LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
-        logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty());
+        logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty());
 
         assertEquals(log101 + log100 + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8));
     }
@@ -87,11 +87,20 @@ public class LogReaderTest {
     void logsForSingeNodeIsRetrieved() {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
-        logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.of("node2.com"));
+        logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.of("node2.com"));
 
         assertEquals(log101 + log100 + log200, baos.toString(UTF_8));
     }
 
+    @Test
+    void logsLimitedToMaxLines() {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
+        logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 2, Optional.of("node2.com"));
+
+        assertEquals(log101 + log100, baos.toString(UTF_8));
+    }
+
     private byte[] compress1(String input) throws IOException {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         OutputStream zip = new GZIPOutputStream(baos);
@@ -100,18 +109,9 @@ public class LogReaderTest {
         return baos.toByteArray();
     }
 
-    private byte[] compress2(String input) throws IOException {
+    private byte[] compress2(String input) {
         byte[] data = input.getBytes();
         return new ZstdCompressor().compress(data, 0, data.length);
     }
 
-    private static File newFolder(File root, String... subDirs) throws IOException {
-        String subFolder = String.join("/", subDirs);
-        File result = new File(root, subFolder);
-        if (!result.mkdirs()) {
-            throw new IOException("Couldn't create folders " + root);
-        }
-        return result;
-    }
-
 }