Combined and zipped outputstream

2 files changed, 66 insertions, 15 deletions

diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
index 1f84c9eaac3..e4c6c9a0918 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
@@ -40,6 +40,15 @@ public class LogHandler extends ThreadedHttpRequestHandler {
                 .map(Long::valueOf).map(Instant::ofEpochMilli).orElseGet(Instant::now);
 
         try {
+            if (request.hasProperty("streaming")) {
+                return new HttpResponse(200) {
+                    @Override
+                    public void render(OutputStream outputStream) {
+                        logReader.writeLogs(outputStream, earliestLogThreshold, latestLogThreshold);
+                    }
+                };
+            }
+
             JSONObject logJson = logReader.readLogs(earliestLogThreshold, latestLogThreshold);
             responseJSON.put("logs", logJson);
         } catch (IOException | JSONException e) {
diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
index df289bd906f..b53290a1a08 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
@@ -1,20 +1,29 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.container.handler;
 
+import com.yahoo.collections.Pair;
 import com.yahoo.vespa.defaults.Defaults;
 import org.json.JSONException;
 import org.json.JSONObject;
 
 import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Base64;
-import java.util.Iterator;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.regex.Pattern;
-import java.util.stream.Stream;
+import java.util.stream.Collectors;
+import java.util.zip.GZIPOutputStream;
 
 class LogReader {
 
@@ -30,29 +39,62 @@ class LogReader {
         this.logFilePattern = logFilePattern;
     }
 
+    // TODO: Remove when all clients use streaming
     JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws IOException, JSONException {
         JSONObject json = new JSONObject();
         latestLogThreshold = latestLogThreshold.plus(Duration.ofMinutes(5)); // Add some time to allow retrieving logs currently being modified
-        traverseFolder(logDirectory, json, earliestLogThreshold, latestLogThreshold, "");
+        for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) {
+            StringBuilder filenameBuilder = new StringBuilder();
+            logDirectory.relativize(file).iterator().forEachRemaining(p -> filenameBuilder.append("-").append(p.getFileName().toString()));
+            json.put(filenameBuilder.substring(1), Base64.getEncoder().encodeToString(Files.readAllBytes(file)));
+        }
         return json;
     }
 
-    private void traverseFolder(Path path, JSONObject json, Instant earliestLogThreshold, Instant latestLogThreshold, String filenamePrefix) throws IOException, JSONException {
-        try (Stream<Path> files = Files.list(path)) {
-            for (Iterator<Path> it = files.iterator(); it.hasNext(); ) {
-                Path child = it.next();
-                String filename = child.getFileName().toString();
-                Instant lastModified = Files.getLastModifiedTime(child).toInstant();
-                if (Files.isRegularFile(child)) {
+    void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) {
+        try (OutputStream bos = new GZIPOutputStream(outputStream)) {
+            for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) {
+                Files.copy(file, bos);
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private List<Path> getMatchingFiles(Instant earliestLogThreshold, Instant latestLogThreshold) {
+        final List<Pair<Path, Instant>> paths = new LinkedList<>();
+        try {
+            Files.walkFileTree(logDirectory, new SimpleFileVisitor<>() {
+
+                @Override
+                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+                    Instant lastModified = attrs.lastModifiedTime().toInstant();
                     if (lastModified.isAfter(earliestLogThreshold) &&
                             lastModified.isBefore(latestLogThreshold) &&
-                            logFilePattern.matcher(filename).matches()) {
-                        json.put(filenamePrefix + filename, Base64.getEncoder().encodeToString(Files.readAllBytes(child)));
+                            logFilePattern.matcher(file.getFileName().toString()).matches()) {
+                        paths.add(new Pair<>(file, lastModified));
                     }
-                } else {
-                    traverseFolder(child, json, earliestLogThreshold, latestLogThreshold, filenamePrefix + filename + "-");
+
+                    return FileVisitResult.CONTINUE;
                 }
-            }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
         }
+
+        return paths.stream()
+                .sorted(Comparator.comparing(Pair::getSecond))
+                .map(Pair::getFirst)
+                .collect(Collectors.toList());
     }
 }