diff options
author | Valerij Fredriksen <valerijf@verizonmedia.com> | 2019-03-05 15:52:54 +0100 |
---|---|---|
committer | Valerij Fredriksen <valerijf@verizonmedia.com> | 2019-03-06 12:35:16 +0100 |
commit | b601044040bb3526e5dfaf3688bac48c1b83b625 (patch) | |
tree | b057562ed61119123ba9e114572de997257a267e /container-core | |
parent | b7264e3244c183bb9c266d5d6ad243ab594b93fd (diff) |
Make LogHandler configurable
Diffstat (limited to 'container-core')
6 files changed, 88 insertions, 87 deletions
diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 12d804e86c3..345911a810f 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -94,24 +94,11 @@ "public" ], "methods": [ - "public void <init>(java.util.concurrent.Executor)", - "protected void <init>(java.util.concurrent.Executor, com.yahoo.container.handler.LogReader)", + "public void <init>(java.util.concurrent.Executor, com.yahoo.container.core.LogHandlerConfig)", "public com.yahoo.container.jdisc.HttpResponse handle(com.yahoo.container.jdisc.HttpRequest)" ], "fields": [] }, - "com.yahoo.container.handler.LogReader": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "protected org.json.JSONObject readLogs(java.lang.String, long, long)" - ], - "fields": [] - }, "com.yahoo.container.handler.Prefix": { "superClass": "java.lang.Object", "interfaces": [ diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 81a93d9f6b1..1f84c9eaac3 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -2,6 +2,7 @@ package com.yahoo.container.handler; import com.google.inject.Inject; +import com.yahoo.container.core.LogHandlerConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; @@ -11,21 +12,20 @@ import org.json.JSONObject; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; -import java.util.Arrays; -import java.util.HashMap; +import java.time.Instant; +import java.util.Optional; import java.util.concurrent.Executor; public class LogHandler extends ThreadedHttpRequestHandler { - private static final String LOG_DIRECTORY = "/home/y/logs/vespa/logarchive/"; private final LogReader logReader; @Inject - public LogHandler(Executor executor) { - this(executor, new LogReader()); + public LogHandler(Executor executor, LogHandlerConfig config) { + this(executor, new LogReader(config.logDirectory(), config.logPattern())); } - protected LogHandler(Executor executor, LogReader logReader) { + LogHandler(Executor executor, LogReader logReader) { super(executor); this.logReader = logReader; } @@ -34,11 +34,13 @@ public class LogHandler extends ThreadedHttpRequestHandler { public HttpResponse handle(HttpRequest request) { JSONObject responseJSON = new JSONObject(); - HashMap<String, String> apiParams = getParameters(request); - long earliestLogThreshold = getEarliestThreshold(apiParams); - long latestLogThreshold = getLatestThreshold(apiParams); + Instant earliestLogThreshold = Optional.ofNullable(request.getProperty("from")) + .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MIN); + Instant latestLogThreshold = Optional.ofNullable(request.getProperty("to")) + .map(Long::valueOf).map(Instant::ofEpochMilli).orElseGet(Instant::now); + try { - JSONObject logJson = logReader.readLogs(LOG_DIRECTORY, earliestLogThreshold, latestLogThreshold); + JSONObject logJson = logReader.readLogs(earliestLogThreshold, latestLogThreshold); responseJSON.put("logs", logJson); } catch (IOException | JSONException e) { return new HttpResponse(404) { @@ -46,6 +48,7 @@ public class LogHandler extends ThreadedHttpRequestHandler { public void render(OutputStream outputStream) {} }; } + return new HttpResponse(200) { @Override public void render(OutputStream outputStream) throws IOException { @@ -55,28 +58,4 @@ public class LogHandler extends ThreadedHttpRequestHandler { } }; } - - private HashMap<String, String> getParameters(HttpRequest request) { - String query = request.getUri().getQuery(); - HashMap<String, String> keyValPair = new HashMap<>(); - Arrays.stream(query.split("&")).forEach(pair -> { - String[] splitPair = pair.split("="); - keyValPair.put(splitPair[0], splitPair[1]); - }); - return keyValPair; - } - - private long getEarliestThreshold(HashMap<String, String> map) { - if (map.containsKey("from")) { - return Long.valueOf(map.get("from")); - } - return Long.MIN_VALUE; - } - - private long getLatestThreshold(HashMap<String, String> map) { - if (map.containsKey("to")) { - return Long.valueOf(map.get("to")); - } - return Long.MAX_VALUE; - } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index c6ae20973fb..df289bd906f 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -1,40 +1,58 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; +import com.yahoo.vespa.defaults.Defaults; import org.json.JSONException; import org.json.JSONObject; -import java.time.Duration; -import java.util.Base64; -import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; +import java.util.Base64; +import java.util.Iterator; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +class LogReader { -public class LogReader { + private final Path logDirectory; + private final Pattern logFilePattern; - long earliestLogThreshold; - long latestLogThreshold; + LogReader(String logDirectory, String logFilePattern) { + this(Paths.get(Defaults.getDefaults().underVespaHome(logDirectory)), Pattern.compile(logFilePattern)); + } + + LogReader(Path logDirectory, Pattern logFilePattern) { + this.logDirectory = logDirectory; + this.logFilePattern = logFilePattern; + } - protected JSONObject readLogs(String logDirectory, long earliestLogThreshold, long latestLogThreshold) throws IOException, JSONException { - this.earliestLogThreshold = earliestLogThreshold; - this.latestLogThreshold = latestLogThreshold + Duration.ofMinutes(5).toMillis(); // Add some time to allow retrieving logs currently being modified + JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws IOException, JSONException { JSONObject json = new JSONObject(); - File root = new File(logDirectory); - traverse_folder(root, json, ""); + latestLogThreshold = latestLogThreshold.plus(Duration.ofMinutes(5)); // Add some time to allow retrieving logs currently being modified + traverseFolder(logDirectory, json, earliestLogThreshold, latestLogThreshold, ""); return json; } - private void traverse_folder(File root, JSONObject json, String filename) throws IOException, JSONException { - File[] files = root.listFiles(); - for(File child : files) { - long logTime = Files.getLastModifiedTime(child.toPath()).toMillis(); - if(child.isFile() && earliestLogThreshold < logTime && logTime < latestLogThreshold) { - json.put(filename + child.getName(), Base64.getEncoder().encodeToString(Files.readAllBytes(child.toPath()))); - } - else if (!child.isFile()){ - traverse_folder(child, json, filename + child.getName() + "-"); + private void traverseFolder(Path path, JSONObject json, Instant earliestLogThreshold, Instant latestLogThreshold, String filenamePrefix) throws IOException, JSONException { + try (Stream<Path> files = Files.list(path)) { + for (Iterator<Path> it = files.iterator(); it.hasNext(); ) { + Path child = it.next(); + String filename = child.getFileName().toString(); + Instant lastModified = Files.getLastModifiedTime(child).toInstant(); + if (Files.isRegularFile(child)) { + if (lastModified.isAfter(earliestLogThreshold) && + lastModified.isBefore(latestLogThreshold) && + logFilePattern.matcher(filename).matches()) { + json.put(filenamePrefix + filename, Base64.getEncoder().encodeToString(Files.readAllBytes(child))); + } + } else { + traverseFolder(child, json, earliestLogThreshold, latestLogThreshold, filenamePrefix + filename + "-"); + } } } } - } diff --git a/container-core/src/main/resources/configdefinitions/log-handler.def b/container-core/src/main/resources/configdefinitions/log-handler.def new file mode 100644 index 00000000000..b30d6eebcf3 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/log-handler.def @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=container.core + +## Path to log directory, can be relative or absolute. +## Relative paths will be resolved relative to $VESPA_HOME +logDirectory string default="logs/vespa/logarchive/" + +## File name regex of logs to include +logPattern string default=".*" diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index 365b0313cf8..d578a745c9e 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -9,9 +9,10 @@ import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.time.Instant; import java.util.concurrent.Executor; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; public class LogHandlerTest { @@ -43,9 +44,13 @@ public class LogHandlerTest { } class MockLogReader extends LogReader { + MockLogReader() { + super("", ""); + } + @Override - protected JSONObject readLogs(String logDirectory, long earliestLogThreshold, long latestLogThreshold) throws JSONException { - if(latestLogThreshold > 1000) { + protected JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws JSONException { + if (latestLogThreshold.isAfter(Instant.ofEpochMilli(1000))) { return new JSONObject("{\"one\":\"newer_log\"}"); } else { return new JSONObject("{\"two\":\"older_log\"}"); diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index d7c802faa3b..647dc571e75 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -2,28 +2,23 @@ package com.yahoo.container.handler; import org.json.JSONObject; -import org.junit.Before; import org.junit.Test; -import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Instant; +import java.util.regex.Pattern; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; public class LogReaderTest { - ByteArrayOutputStream outputStream; - - @Before - public void setup() { - outputStream = new ByteArrayOutputStream(); - } + private final Path logDirectory = Paths.get("src/test/resources/logfolder/"); @Test public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{ - String logDirectory = "src/test/resources/logfolder/"; - LogReader logReader = new LogReader(); - JSONObject json = logReader.readLogs(logDirectory, 21, Instant.now().toEpochMilli()); + LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); + JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}"; String actual = json.toString(); assertEquals(expected, actual); @@ -31,11 +26,19 @@ public class LogReaderTest { @Test public void testThatLogsOutsideRangeAreExcluded() throws Exception { - String logDirectory = "src/test/resources/logfolder/"; - LogReader logReader = new LogReader(); - JSONObject json = logReader.readLogs(logDirectory, Long.MAX_VALUE, Long.MIN_VALUE); + LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); + JSONObject json = logReader.readLogs(Instant.MAX, Instant.MIN); String expected = "{}"; String actual = json.toString(); assertEquals(expected, actual); } + + @Test + public void testThatLogsNotMatchingRegexAreExcluded() throws Exception { + LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*2\\.log")); + JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); + String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"}"; + String actual = json.toString(); + assertEquals(expected, actual); + } } |