Check if log file is zipped (#8773)

* Check if log file is zipped

* Check that outputStream isn't already a GZIPOutputStream

* No longer depend on file type order

2 files changed, 36 insertions, 6 deletions

diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
index b53290a1a08..95a0e9a6766 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
@@ -6,6 +6,7 @@ import com.yahoo.vespa.defaults.Defaults;
 import org.json.JSONException;
 import org.json.JSONObject;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.UncheckedIOException;
@@ -23,6 +24,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
 class LogReader {
@@ -46,16 +48,24 @@ class LogReader {
         for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) {
             StringBuilder filenameBuilder = new StringBuilder();
             logDirectory.relativize(file).iterator().forEachRemaining(p -> filenameBuilder.append("-").append(p.getFileName().toString()));
-            json.put(filenameBuilder.substring(1), Base64.getEncoder().encodeToString(Files.readAllBytes(file)));
+            byte[] fileData = file.toString().endsWith(".gz") ? new GZIPInputStream(new ByteArrayInputStream(Files.readAllBytes(file))).readAllBytes() : Files.readAllBytes(file);
+            json.put(filenameBuilder.substring(1), Base64.getEncoder().encodeToString(fileData));
         }
         return json;
     }
 
     void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) {
-        try (OutputStream bos = new GZIPOutputStream(outputStream)) {
+        try {
             for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) {
-                Files.copy(file, bos);
+                if (file.toString().endsWith(".gz")) {
+                    Files.copy(file, outputStream);
+                } else {
+                    OutputStream zip = new GZIPOutputStream(outputStream);
+                    Files.copy(file, zip);
+                    zip.close();
+                }
             }
+            outputStream.close();
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
index c919aec9907..1bd68ad0c48 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
@@ -9,6 +9,7 @@ import org.junit.Test;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -17,6 +18,7 @@ import java.time.Instant;
 import java.util.Scanner;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 
 import static org.junit.Assert.assertEquals;
 
@@ -30,7 +32,7 @@ public class LogReaderTest {
         Files.createDirectories(logDirectory.resolve("subfolder"));
 
         Files.setLastModifiedTime(
-                Files.write(logDirectory.resolve("log1.log"), "This is one log file\n".getBytes()),
+                Files.write(logDirectory.resolve("log1.log.gz"), compress("This is one log file\n")),
                 FileTime.from(Instant.ofEpochMilli(123)));
         Files.setLastModifiedTime(
                 Files.write(logDirectory.resolve("subfolder/log2.log"), "This is another log file\n".getBytes()),
@@ -41,7 +43,7 @@ public class LogReaderTest {
     public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{
         LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
         JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now());
-        String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGUK\"}";
+        String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\",\"log1.log.gz\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGUK\"}";
         String actual = json.toString();
         assertEquals(expected, actual);
     }
@@ -66,6 +68,15 @@ public class LogReaderTest {
 
     @Test
     public void testZippedStreaming() throws IOException {
+
+        // Add some more files
+        Files.setLastModifiedTime(
+                Files.write(logDirectory.resolve("log3.gz"), compress("Three\n")),
+                FileTime.from(Instant.ofEpochMilli(324)));
+        Files.setLastModifiedTime(
+                Files.write(logDirectory.resolve("log4"), "Four\n".getBytes()),
+                FileTime.from(Instant.ofEpochMilli(432)));
+
         ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream();
         LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
         logReader.writeLogs(zippedBaos, Instant.ofEpochMilli(21), Instant.now());
@@ -74,7 +85,16 @@ public class LogReaderTest {
         Scanner s = new Scanner(unzippedIs).useDelimiter("\\A");
         String actual = s.hasNext() ? s.next() : "";
 
-        String expected = "This is one log file\nThis is another log file\n";
+        String expected = "This is one log file\nThis is another log file\nThree\nFour\n";
         assertEquals(expected, actual);
     }
+
+    private byte[] compress(String input) throws IOException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        OutputStream zip = new GZIPOutputStream(baos);
+        zip.write(input.getBytes());
+        zip.close();
+        return baos.toByteArray();
+    }
+
 }