diff options
author | Ola Aunrønning <olaa@oath.com> | 2019-04-05 11:08:38 +0200 |
---|---|---|
committer | Ola Aunrønning <olaa@oath.com> | 2019-04-05 11:34:57 +0200 |
commit | 190b9c4ce97664f90425a90fd759d4d5fc51989c (patch) | |
tree | 1fc25b5b910f4240b9e2e19f70c1d31d7b8bfd7f | |
parent | 8a2cb4a349de1f7cc44f7e36a1a40c824e39912b (diff) |
Remove old log reading
4 files changed, 30 insertions, 87 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index f6bdeb02b9c..645c231531d 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -6,12 +6,8 @@ import com.yahoo.container.core.LogHandlerConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import org.json.JSONException; -import org.json.JSONObject; -import java.io.IOException; import java.io.OutputStream; -import java.io.OutputStreamWriter; import java.time.Instant; import java.util.Optional; import java.util.concurrent.Executor; @@ -32,47 +28,19 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public HttpResponse handle(HttpRequest request) { - JSONObject responseJSON = new JSONObject(); Instant earliestLogThreshold = Optional.ofNullable(request.getProperty("from")) .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MIN); Instant latestLogThreshold = Optional.ofNullable(request.getProperty("to")) .map(Long::valueOf).map(Instant::ofEpochMilli).orElseGet(Instant::now); - try { - if (request.hasProperty("streaming")) { - return new HttpResponse(200) { - { - headers().add("Content-Encoding", "gzip"); - } - - @Override - public void render(OutputStream outputStream) { - logReader.writeLogs(outputStream, earliestLogThreshold, latestLogThreshold); - } - }; - } - - JSONObject logJson = logReader.readLogs(earliestLogThreshold, latestLogThreshold); - responseJSON.put("logs", logJson); - } catch (IOException | JSONException e) { - return new HttpResponse(404) { - @Override - public void render(OutputStream outputStream) {} - }; - } - return new HttpResponse(200) { - @Override - public void render(OutputStream outputStream) throws IOException { - OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream); - outputStreamWriter.write(responseJSON.toString()); - outputStreamWriter.close(); + { + headers().add("Content-Encoding", "gzip"); } - @Override - public String getContentType() { - return "application/json"; + public void render(OutputStream outputStream) { + logReader.writeLogs(outputStream, earliestLogThreshold, latestLogThreshold); } }; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 663741f9bef..6cb92244522 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -3,10 +3,7 @@ package com.yahoo.container.handler; import com.yahoo.collections.Pair; import com.yahoo.vespa.defaults.Defaults; -import org.json.JSONException; -import org.json.JSONObject; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; @@ -16,15 +13,12 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.time.Duration; import java.time.Instant; -import java.util.Base64; import java.util.Comparator; import java.util.LinkedList; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; class LogReader { @@ -41,19 +35,6 @@ class LogReader { this.logFilePattern = logFilePattern; } - // TODO: Remove when all clients use streaming - JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws IOException, JSONException { - JSONObject json = new JSONObject(); - latestLogThreshold = latestLogThreshold.plus(Duration.ofMinutes(5)); // Add some time to allow retrieving logs currently being modified - for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) { - StringBuilder filenameBuilder = new StringBuilder(); - logDirectory.relativize(file).iterator().forEachRemaining(p -> filenameBuilder.append("-").append(p.getFileName().toString())); - byte[] fileData = file.toString().endsWith(".gz") ? new GZIPInputStream(new ByteArrayInputStream(Files.readAllBytes(file))).readAllBytes() : Files.readAllBytes(file); - json.put(filenameBuilder.substring(1), Base64.getEncoder().encodeToString(fileData)); - } - return json; - } - void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) { try { for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) { diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index d578a745c9e..f0885451431 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -3,12 +3,11 @@ package com.yahoo.container.handler; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import org.json.JSONException; -import org.json.JSONObject; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.time.Instant; import java.util.concurrent.Executor; @@ -17,7 +16,6 @@ import static org.mockito.Mockito.mock; public class LogHandlerTest { - @Test public void handleCorrectlyParsesQueryParameters() throws IOException { MockLogReader mockLogReader = new MockLogReader(); @@ -28,7 +26,7 @@ public class LogHandlerTest { HttpResponse response = logHandler.handle(HttpRequest.createTestRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.GET)); ByteArrayOutputStream bos = new ByteArrayOutputStream(); response.render(bos); - String expectedResponse = "{\"logs\":{\"one\":\"newer_log\"}}"; + String expectedResponse = "newer log"; assertEquals(expectedResponse, bos.toString()); } @@ -37,7 +35,7 @@ public class LogHandlerTest { HttpResponse response = logHandler.handle(HttpRequest.createTestRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.GET)); ByteArrayOutputStream bos = new ByteArrayOutputStream(); response.render(bos); - String expectedResponse = "{\"logs\":{\"two\":\"older_log\"}}"; + String expectedResponse = "older log"; assertEquals(expectedResponse, bos.toString()); } @@ -49,12 +47,14 @@ public class LogHandlerTest { } @Override - protected JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws JSONException { - if (latestLogThreshold.isAfter(Instant.ofEpochMilli(1000))) { - return new JSONObject("{\"one\":\"newer_log\"}"); - } else { - return new JSONObject("{\"two\":\"older_log\"}"); - } + protected void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) { + try { + if (latestLogThreshold.isAfter(Instant.ofEpochMilli(1000))) { + outputStream.write("newer log".getBytes()); + } else { + outputStream.write("older log".getBytes()); + } + } catch (Exception e) {} } } } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index 464d6f772eb..27300e5f3f5 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -2,7 +2,6 @@ package com.yahoo.container.handler; import com.yahoo.vespa.test.file.TestFileSystem; -import org.json.JSONObject; import org.junit.Before; import org.junit.Test; @@ -15,7 +14,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.FileTime; import java.time.Instant; -import java.util.Scanner; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -37,47 +35,37 @@ public class LogReaderTest { Files.setLastModifiedTime( Files.write(logDirectory.resolve("subfolder/log2.log"), "This is another log file\n".getBytes()), FileTime.from(Instant.ofEpochMilli(234))); - } - @Test - public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{ - LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); - String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\",\"log1.log.gz\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGUK\"}"; - String actual = json.toString(); - assertEquals(expected, actual); } @Test public void testThatLogsOutsideRangeAreExcluded() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - JSONObject json = logReader.readLogs(Instant.MAX, Instant.MIN); - String expected = "{}"; - String actual = json.toString(); + logReader.writeLogs(baos, Instant.ofEpochMilli(235), Instant.ofEpochMilli(300)); + String expected = ""; + String actual = decompress(baos.toByteArray()); assertEquals(expected, actual); } @Test public void testThatLogsNotMatchingRegexAreExcluded() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*2\\.log")); - JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); - String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\"}"; - String actual = json.toString(); + logReader.writeLogs(baos, Instant.ofEpochMilli(21), Instant.now()); + String expected = "This is another log file\n"; + String actual = decompress(baos.toByteArray()); assertEquals(expected, actual); } @Test public void testZippedStreaming() throws IOException { - ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); logReader.writeLogs(zippedBaos, Instant.ofEpochMilli(21), Instant.now()); - GZIPInputStream unzippedIs = new GZIPInputStream(new ByteArrayInputStream(zippedBaos.toByteArray())); - - Scanner s = new Scanner(unzippedIs).useDelimiter("\\A"); - String actual = s.hasNext() ? s.next() : ""; String expected = "This is one log file\nThis is another log file\n"; + String actual = decompress(zippedBaos.toByteArray()); assertEquals(expected, actual); } @@ -89,4 +77,10 @@ public class LogReaderTest { return baos.toByteArray(); } + private String decompress(byte[] input) throws IOException { + if (input.length == 0) return ""; + byte[] decompressed = new GZIPInputStream(new ByteArrayInputStream(input)).readAllBytes(); + return new String(decompressed); + } + } |