Combined and zipped outputstream

5 files changed, 89 insertions, 19 deletions

diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
index 1f84c9eaac3..e4c6c9a0918 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
@@ -40,6 +40,15 @@ public class LogHandler extends ThreadedHttpRequestHandler {
                 .map(Long::valueOf).map(Instant::ofEpochMilli).orElseGet(Instant::now);
 
         try {
+            if (request.hasProperty("streaming")) {
+                return new HttpResponse(200) {
+                    @Override
+                    public void render(OutputStream outputStream) {
+                        logReader.writeLogs(outputStream, earliestLogThreshold, latestLogThreshold);
+                    }
+                };
+            }
+
             JSONObject logJson = logReader.readLogs(earliestLogThreshold, latestLogThreshold);
             responseJSON.put("logs", logJson);
         } catch (IOException | JSONException e) {
diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
index df289bd906f..b53290a1a08 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
@@ -1,20 +1,29 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.container.handler;
 
+import com.yahoo.collections.Pair;
 import com.yahoo.vespa.defaults.Defaults;
 import org.json.JSONException;
 import org.json.JSONObject;
 
 import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Base64;
-import java.util.Iterator;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.regex.Pattern;
-import java.util.stream.Stream;
+import java.util.stream.Collectors;
+import java.util.zip.GZIPOutputStream;
 
 class LogReader {
 
@@ -30,29 +39,62 @@ class LogReader {
         this.logFilePattern = logFilePattern;
     }
 
+    // TODO: Remove when all clients use streaming
     JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws IOException, JSONException {
         JSONObject json = new JSONObject();
         latestLogThreshold = latestLogThreshold.plus(Duration.ofMinutes(5)); // Add some time to allow retrieving logs currently being modified
-        traverseFolder(logDirectory, json, earliestLogThreshold, latestLogThreshold, "");
+        for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) {
+            StringBuilder filenameBuilder = new StringBuilder();
+            logDirectory.relativize(file).iterator().forEachRemaining(p -> filenameBuilder.append("-").append(p.getFileName().toString()));
+            json.put(filenameBuilder.substring(1), Base64.getEncoder().encodeToString(Files.readAllBytes(file)));
+        }
         return json;
     }
 
-    private void traverseFolder(Path path, JSONObject json, Instant earliestLogThreshold, Instant latestLogThreshold, String filenamePrefix) throws IOException, JSONException {
-        try (Stream<Path> files = Files.list(path)) {
-            for (Iterator<Path> it = files.iterator(); it.hasNext(); ) {
-                Path child = it.next();
-                String filename = child.getFileName().toString();
-                Instant lastModified = Files.getLastModifiedTime(child).toInstant();
-                if (Files.isRegularFile(child)) {
+    void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) {
+        try (OutputStream bos = new GZIPOutputStream(outputStream)) {
+            for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) {
+                Files.copy(file, bos);
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private List<Path> getMatchingFiles(Instant earliestLogThreshold, Instant latestLogThreshold) {
+        final List<Pair<Path, Instant>> paths = new LinkedList<>();
+        try {
+            Files.walkFileTree(logDirectory, new SimpleFileVisitor<>() {
+
+                @Override
+                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+                    Instant lastModified = attrs.lastModifiedTime().toInstant();
                     if (lastModified.isAfter(earliestLogThreshold) &&
                             lastModified.isBefore(latestLogThreshold) &&
-                            logFilePattern.matcher(filename).matches()) {
-                        json.put(filenamePrefix + filename, Base64.getEncoder().encodeToString(Files.readAllBytes(child)));
+                            logFilePattern.matcher(file.getFileName().toString()).matches()) {
+                        paths.add(new Pair<>(file, lastModified));
                     }
-                } else {
-                    traverseFolder(child, json, earliestLogThreshold, latestLogThreshold, filenamePrefix + filename + "-");
+
+                    return FileVisitResult.CONTINUE;
                 }
-            }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
         }
+
+        return paths.stream()
+                .sorted(Comparator.comparing(Pair::getSecond))
+                .map(Pair::getFirst)
+                .collect(Collectors.toList());
     }
 }
diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
index 647dc571e75..1a82bbb5fff 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
@@ -4,10 +4,15 @@ package com.yahoo.container.handler;
 import org.json.JSONObject;
 import org.junit.Test;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Instant;
+import java.util.Scanner;
 import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
 
 import static org.junit.Assert.assertEquals;
 
@@ -19,7 +24,7 @@ public class LogReaderTest {
     public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{
         LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
         JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now());
-        String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}";
+        String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGUK\"}";
         String actual = json.toString();
         assertEquals(expected, actual);
     }
@@ -37,8 +42,22 @@ public class LogReaderTest {
     public void testThatLogsNotMatchingRegexAreExcluded() throws Exception {
         LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*2\\.log"));
         JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now());
-        String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"}";
+        String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\"}";
         String actual = json.toString();
         assertEquals(expected, actual);
     }
+
+    @Test
+    public void testZippedStreaming() throws IOException {
+        ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream();
+        LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
+        logReader.writeLogs(zippedBaos, Instant.ofEpochMilli(21), Instant.now());
+        GZIPInputStream unzippedIs = new GZIPInputStream(new ByteArrayInputStream(zippedBaos.toByteArray()));
+
+        Scanner s = new Scanner(unzippedIs).useDelimiter("\\A");
+        String actual = s.hasNext() ? s.next() : "";
+
+        String expected = "This is one log file\nThis is another log file\n";
+        assertEquals(expected, actual);
+    }
 }
diff --git a/container-core/src/test/resources/logfolder/log1.log b/container-core/src/test/resources/logfolder/log1.log
index bb85d5a4950..081a0a53b14 100644
--- a/container-core/src/test/resources/logfolder/log1.log
+++ b/container-core/src/test/resources/logfolder/log1.log
@@ -1 +1 @@
-This is one log file
\ No newline at end of file
+This is one log file
diff --git a/container-core/src/test/resources/logfolder/subfolder/log2.log b/container-core/src/test/resources/logfolder/subfolder/log2.log
index aee6eaca2e8..3b5f20ded2b 100644
--- a/container-core/src/test/resources/logfolder/subfolder/log2.log
+++ b/container-core/src/test/resources/logfolder/subfolder/log2.log
@@ -1 +1 @@
-This is another log file
\ No newline at end of file
+This is another log file