diff options
author | Valerij Fredriksen <valerijf@verizonmedia.com> | 2019-03-06 14:22:51 +0100 |
---|---|---|
committer | Valerij Fredriksen <valerijf@verizonmedia.com> | 2019-03-06 14:37:00 +0100 |
commit | 2c3e6aa7cd0135465f448b4abe48c32043241822 (patch) | |
tree | 8dbb7f8acb5d99e0e308e60fda9cb54dac528550 /container-core | |
parent | b601044040bb3526e5dfaf3688bac48c1b83b625 (diff) |
Combined and zipped outputstream
Diffstat (limited to 'container-core')
5 files changed, 89 insertions, 19 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 1f84c9eaac3..e4c6c9a0918 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -40,6 +40,15 @@ public class LogHandler extends ThreadedHttpRequestHandler { .map(Long::valueOf).map(Instant::ofEpochMilli).orElseGet(Instant::now); try { + if (request.hasProperty("streaming")) { + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) { + logReader.writeLogs(outputStream, earliestLogThreshold, latestLogThreshold); + } + }; + } + JSONObject logJson = logReader.readLogs(earliestLogThreshold, latestLogThreshold); responseJSON.put("logs", logJson); } catch (IOException | JSONException e) { diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index df289bd906f..b53290a1a08 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -1,20 +1,29 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; +import com.yahoo.collections.Pair; import com.yahoo.vespa.defaults.Defaults; import org.json.JSONException; import org.json.JSONObject; import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; import java.util.Base64; -import java.util.Iterator; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; import java.util.regex.Pattern; -import java.util.stream.Stream; +import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; class LogReader { @@ -30,29 +39,62 @@ class LogReader { this.logFilePattern = logFilePattern; } + // TODO: Remove when all clients use streaming JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws IOException, JSONException { JSONObject json = new JSONObject(); latestLogThreshold = latestLogThreshold.plus(Duration.ofMinutes(5)); // Add some time to allow retrieving logs currently being modified - traverseFolder(logDirectory, json, earliestLogThreshold, latestLogThreshold, ""); + for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) { + StringBuilder filenameBuilder = new StringBuilder(); + logDirectory.relativize(file).iterator().forEachRemaining(p -> filenameBuilder.append("-").append(p.getFileName().toString())); + json.put(filenameBuilder.substring(1), Base64.getEncoder().encodeToString(Files.readAllBytes(file))); + } return json; } - private void traverseFolder(Path path, JSONObject json, Instant earliestLogThreshold, Instant latestLogThreshold, String filenamePrefix) throws IOException, JSONException { - try (Stream<Path> files = Files.list(path)) { - for (Iterator<Path> it = files.iterator(); it.hasNext(); ) { - Path child = it.next(); - String filename = child.getFileName().toString(); - Instant lastModified = Files.getLastModifiedTime(child).toInstant(); - if (Files.isRegularFile(child)) { + void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) { + try (OutputStream bos = new GZIPOutputStream(outputStream)) { + for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) { + Files.copy(file, bos); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private List<Path> getMatchingFiles(Instant earliestLogThreshold, Instant latestLogThreshold) { + final List<Pair<Path, Instant>> paths = new LinkedList<>(); + try { + Files.walkFileTree(logDirectory, new SimpleFileVisitor<>() { + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + Instant lastModified = attrs.lastModifiedTime().toInstant(); if (lastModified.isAfter(earliestLogThreshold) && lastModified.isBefore(latestLogThreshold) && - logFilePattern.matcher(filename).matches()) { - json.put(filenamePrefix + filename, Base64.getEncoder().encodeToString(Files.readAllBytes(child))); + logFilePattern.matcher(file.getFileName().toString()).matches()) { + paths.add(new Pair<>(file, lastModified)); } - } else { - traverseFolder(child, json, earliestLogThreshold, latestLogThreshold, filenamePrefix + filename + "-"); + + return FileVisitResult.CONTINUE; } - } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + throw new UncheckedIOException(e); } + + return paths.stream() + .sorted(Comparator.comparing(Pair::getSecond)) + .map(Pair::getFirst) + .collect(Collectors.toList()); } } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index 647dc571e75..1a82bbb5fff 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -4,10 +4,15 @@ package com.yahoo.container.handler; import org.json.JSONObject; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; +import java.util.Scanner; import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; import static org.junit.Assert.assertEquals; @@ -19,7 +24,7 @@ public class LogReaderTest { public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{ LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); - String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}"; + String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGUK\"}"; String actual = json.toString(); assertEquals(expected, actual); } @@ -37,8 +42,22 @@ public class LogReaderTest { public void testThatLogsNotMatchingRegexAreExcluded() throws Exception { LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*2\\.log")); JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); - String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"}"; + String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\"}"; String actual = json.toString(); assertEquals(expected, actual); } + + @Test + public void testZippedStreaming() throws IOException { + ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); + LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); + logReader.writeLogs(zippedBaos, Instant.ofEpochMilli(21), Instant.now()); + GZIPInputStream unzippedIs = new GZIPInputStream(new ByteArrayInputStream(zippedBaos.toByteArray())); + + Scanner s = new Scanner(unzippedIs).useDelimiter("\\A"); + String actual = s.hasNext() ? s.next() : ""; + + String expected = "This is one log file\nThis is another log file\n"; + assertEquals(expected, actual); + } } diff --git a/container-core/src/test/resources/logfolder/log1.log b/container-core/src/test/resources/logfolder/log1.log index bb85d5a4950..081a0a53b14 100644 --- a/container-core/src/test/resources/logfolder/log1.log +++ b/container-core/src/test/resources/logfolder/log1.log @@ -1 +1 @@ -This is one log file
\ No newline at end of file +This is one log file diff --git a/container-core/src/test/resources/logfolder/subfolder/log2.log b/container-core/src/test/resources/logfolder/subfolder/log2.log index aee6eaca2e8..3b5f20ded2b 100644 --- a/container-core/src/test/resources/logfolder/subfolder/log2.log +++ b/container-core/src/test/resources/logfolder/subfolder/log2.log @@ -1 +1 @@ -This is another log file
\ No newline at end of file +This is another log file |