diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2022-09-21 14:21:54 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-09-21 14:21:54 +0200 |
commit | 75430aac3718930f640d6594ccfcf959ec104d05 (patch) | |
tree | 62974ecc727dfead76ea08f7235656065c3e4b55 | |
parent | ec52029ecb619e0a40691b0860591b94273268fa (diff) | |
parent | 5832b4ab5841e1b0b04dfeb88bc75a919502e25b (diff) |
Merge pull request #24154 from vespa-engine/freva/max-log-lines
Make it possible to limit number of log lines returned
4 files changed, 23 insertions, 36 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index d151eb6862d..72a399744f3 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -4,21 +4,15 @@ package com.yahoo.container.handler; import com.yahoo.component.annotation.Inject; import com.yahoo.container.core.LogHandlerConfig; import com.yahoo.container.jdisc.AsyncHttpResponse; -import com.yahoo.container.jdisc.ContentChannelOutputStream; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; -import java.io.IOException; -import java.io.InterruptedIOException; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.time.Instant; import java.util.Optional; import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; public class LogHandler extends ThreadedHttpRequestHandler { @@ -42,6 +36,8 @@ public class LogHandler extends ThreadedHttpRequestHandler { .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MIN); Instant to = Optional.ofNullable(request.getProperty("to")) .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MAX); + long maxLines = Optional.ofNullable(request.getProperty("maxLines")) + .map(Long::valueOf).orElse(100_000L); Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname")); return new AsyncHttpResponse(200) { @@ -50,7 +46,7 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public void render(OutputStream output, ContentChannel networkChannel, CompletionHandler handler) { try (output) { - logReader.writeLogs(output, from, to, hostname); + logReader.writeLogs(output, from, to, maxLines, hostname); } catch (Throwable t) { log.log(Level.WARNING, "Failed reading logs from " + from + " to " + to, t); @@ -62,6 +58,4 @@ public class LogHandler extends ThreadedHttpRequestHandler { }; } - - } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 93881b52eb6..dae4b940307 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -59,9 +59,10 @@ class LogReader { this.logFilePattern = logFilePattern; } - void writeLogs(OutputStream out, Instant from, Instant to, Optional<String> hostname) { + void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Optional<String> hostname) { double fromSeconds = from.getEpochSecond() + from.getNano() / 1e9; double toSeconds = to.getEpochSecond() + to.getNano() / 1e9; + long linesWritten = 0; BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out)); for (List<Path> logs : getMatchingFiles(from, to)) { List<LogLineIterator> logLineIterators = new ArrayList<>(); @@ -73,6 +74,7 @@ class LogReader { Iterator<LineWithTimestamp> lines = Iterators.mergeSorted(logLineIterators, Comparator.comparingDouble(LineWithTimestamp::timestamp)); while (lines.hasNext()) { + if (linesWritten++ >= maxLines) return; String line = lines.next().line(); writer.write(line); writer.newLine(); @@ -187,16 +189,7 @@ class LogReader { } - private static class LineWithTimestamp { - final String line; - final double timestamp; - LineWithTimestamp(String line, double timestamp) { - this.line = line; - this.timestamp = timestamp; - } - String line() { return line; } - double timestamp() { return timestamp; } - } + private record LineWithTimestamp(String line, double timestamp) { } /** Returns log files which may have relevant entries, grouped and sorted by {@link #extractTimestamp(Path)} — the first and last group must be filtered. */ private List<List<Path>> getMatchingFiles(Instant from, Instant to) { diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index 3d9a2360e77..8b698b25ab2 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -51,7 +51,7 @@ public class LogHandlerTest { } @Override - protected void writeLogs(OutputStream out, Instant from, Instant to, Optional<String> hostname) { + protected void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Optional<String> hostname) { try { if (to.isAfter(Instant.ofEpochMilli(1000))) { out.write("newer log".getBytes()); diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index 106b8cef35f..e98f8cac276 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -37,7 +37,7 @@ public class LogReaderTest { @BeforeEach public void setup() throws IOException { - logDirectory = newFolder(folder, "opt/vespa/logs").toPath(); + logDirectory = Files.createDirectories(folder.toPath().resolve("opt/vespa/logs")); // Log archive paths and file names indicate what hour they contain logs for, with the start of that hour. // Multiple entries may exist for each hour. Files.createDirectories(logDirectory.resolve("1970/01/01")); @@ -59,7 +59,7 @@ public class LogReaderTest { void testThatLogsOutsideRangeAreExcluded() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), Optional.empty()); + logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), 100, Optional.empty()); assertEquals(log100 + logv11 + log110, baos.toString(UTF_8)); } @@ -68,7 +68,7 @@ public class LogReaderTest { void testThatLogsNotMatchingRegexAreExcluded() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*")); - logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty()); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty()); assertEquals(log101 + logv11, baos.toString(UTF_8)); } @@ -78,7 +78,7 @@ public class LogReaderTest { void testZippedStreaming() { ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty()); + logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty()); assertEquals(log101 + log100 + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8)); } @@ -87,11 +87,20 @@ public class LogReaderTest { void logsForSingeNodeIsRetrieved() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.of("node2.com")); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.of("node2.com")); assertEquals(log101 + log100 + log200, baos.toString(UTF_8)); } + @Test + void logsLimitedToMaxLines() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 2, Optional.of("node2.com")); + + assertEquals(log101 + log100, baos.toString(UTF_8)); + } + private byte[] compress1(String input) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream zip = new GZIPOutputStream(baos); @@ -100,18 +109,9 @@ public class LogReaderTest { return baos.toByteArray(); } - private byte[] compress2(String input) throws IOException { + private byte[] compress2(String input) { byte[] data = input.getBytes(); return new ZstdCompressor().compress(data, 0, data.length); } - private static File newFolder(File root, String... subDirs) throws IOException { - String subFolder = String.join("/", subDirs); - File result = new File(root, subFolder); - if (!result.mkdirs()) { - throw new IOException("Couldn't create folders " + root); - } - return result; - } - } |