diff options
author | olaaun <olaa@oath.com> | 2019-03-13 16:15:12 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-03-13 16:15:12 +0100 |
commit | 8e00586fc4f9828690ff50d31c05c036a0231005 (patch) | |
tree | 183b7965f4c935dd0e3e67f3cca98516c90449b0 /container-core | |
parent | b552a4ee99e256804829e062cd75280d5df3319c (diff) |
Check if log file is zipped (#8773)
* Check if log file is zipped
* Check that outputStream isn't already a GZIPOutputStream
* No longer depend on file type order
Diffstat (limited to 'container-core')
-rw-r--r-- | container-core/src/main/java/com/yahoo/container/handler/LogReader.java | 16 | ||||
-rw-r--r-- | container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java | 26 |
2 files changed, 36 insertions, 6 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index b53290a1a08..95a0e9a6766 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.defaults.Defaults; import org.json.JSONException; import org.json.JSONObject; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; @@ -23,6 +24,7 @@ import java.util.LinkedList; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; class LogReader { @@ -46,16 +48,24 @@ class LogReader { for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) { StringBuilder filenameBuilder = new StringBuilder(); logDirectory.relativize(file).iterator().forEachRemaining(p -> filenameBuilder.append("-").append(p.getFileName().toString())); - json.put(filenameBuilder.substring(1), Base64.getEncoder().encodeToString(Files.readAllBytes(file))); + byte[] fileData = file.toString().endsWith(".gz") ? new GZIPInputStream(new ByteArrayInputStream(Files.readAllBytes(file))).readAllBytes() : Files.readAllBytes(file); + json.put(filenameBuilder.substring(1), Base64.getEncoder().encodeToString(fileData)); } return json; } void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) { - try (OutputStream bos = new GZIPOutputStream(outputStream)) { + try { for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) { - Files.copy(file, bos); + if (file.toString().endsWith(".gz")) { + Files.copy(file, outputStream); + } else { + OutputStream zip = new GZIPOutputStream(outputStream); + Files.copy(file, zip); + zip.close(); + } } + outputStream.close(); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index c919aec9907..1bd68ad0c48 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -9,6 +9,7 @@ import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; @@ -17,6 +18,7 @@ import java.time.Instant; import java.util.Scanner; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import static org.junit.Assert.assertEquals; @@ -30,7 +32,7 @@ public class LogReaderTest { Files.createDirectories(logDirectory.resolve("subfolder")); Files.setLastModifiedTime( - Files.write(logDirectory.resolve("log1.log"), "This is one log file\n".getBytes()), + Files.write(logDirectory.resolve("log1.log.gz"), compress("This is one log file\n")), FileTime.from(Instant.ofEpochMilli(123))); Files.setLastModifiedTime( Files.write(logDirectory.resolve("subfolder/log2.log"), "This is another log file\n".getBytes()), @@ -41,7 +43,7 @@ public class LogReaderTest { public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{ LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); - String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGUK\"}"; + String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\",\"log1.log.gz\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGUK\"}"; String actual = json.toString(); assertEquals(expected, actual); } @@ -66,6 +68,15 @@ public class LogReaderTest { @Test public void testZippedStreaming() throws IOException { + + // Add some more files + Files.setLastModifiedTime( + Files.write(logDirectory.resolve("log3.gz"), compress("Three\n")), + FileTime.from(Instant.ofEpochMilli(324))); + Files.setLastModifiedTime( + Files.write(logDirectory.resolve("log4"), "Four\n".getBytes()), + FileTime.from(Instant.ofEpochMilli(432))); + ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); logReader.writeLogs(zippedBaos, Instant.ofEpochMilli(21), Instant.now()); @@ -74,7 +85,16 @@ public class LogReaderTest { Scanner s = new Scanner(unzippedIs).useDelimiter("\\A"); String actual = s.hasNext() ? s.next() : ""; - String expected = "This is one log file\nThis is another log file\n"; + String expected = "This is one log file\nThis is another log file\nThree\nFour\n"; assertEquals(expected, actual); } + + private byte[] compress(String input) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + OutputStream zip = new GZIPOutputStream(baos); + zip.write(input.getBytes()); + zip.close(); + return baos.toByteArray(); + } + } |