Merge pull request #6993 from vespa-engine/olaa/filter-logs-on-time

Olaa/filter logs on time

12 files changed, 125 insertions, 39 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 6a55fb77933..6c67f730f60 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -483,10 +483,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
 
     // ---------------- Logs ----------------------------------------------------------------
 
-    public HttpResponse getLogs(ApplicationId applicationId) {
-        String logServerHostName = getLogServerURI(applicationId);
+    public HttpResponse getLogs(ApplicationId applicationId, String apiParams) {
+        String logServerURI = getLogServerURI(applicationId) + apiParams;
         LogRetriever logRetriever = new LogRetriever();
-        return logRetriever.getLogs(logServerHostName);
+        return logRetriever.getLogs(logServerURI);
     }
 
     // ---------------- Session operations ----------------------------------------------------------------
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index b65cb370f93..528575f4f27 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -97,7 +97,9 @@ public class ApplicationHandler extends HttpHandler {
         }
 
         if (isLogRequest(request)) {
-            return applicationRepository.getLogs(applicationId);
+            String apiParams = request.getUri().getQuery();
+            apiParams = apiParams == null ? "" : "?" + apiParams;
+            return applicationRepository.getLogs(applicationId, apiParams);
         }
 
         return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index 120119f35bb..2ae8917e905 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -118,7 +118,7 @@ public class ApplicationRepositoryTest {
     }
 
     @Test
-    public void getLogs(){
+    public void getLogs() {
         WireMockServer wireMock = new WireMockServer(wireMockConfig().port(8080));
         wireMock.start();
         WireMock.configureFor("localhost", wireMock.port());
@@ -127,15 +127,15 @@ public class ApplicationRepositoryTest {
                         .withStatus(200)));
         wireMock.start();
         deployApp(testAppLogServerWithContainer);
-        HttpResponse response = applicationRepository.getLogs(applicationId());
-        assertEquals(response.getStatus(),200);
+        HttpResponse response = applicationRepository.getLogs(applicationId(), "");
+        assertEquals(200, response.getStatus());
         wireMock.stop();
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void getLogsNoContainerOnLogServerHostShouldThrowException() {
         deployApp(testApp);
-        applicationRepository.getLogs(applicationId());
+        applicationRepository.getLogs(applicationId(), "");
     }
 
     @Test
diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
index 4183b642af1..4c12bacf145 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
@@ -10,11 +10,13 @@ import org.json.JSONObject;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
+import java.util.Arrays;
+import java.util.HashMap;
 import java.util.concurrent.Executor;
 
 public class LogHandler extends ThreadedHttpRequestHandler {
 
-    private static final String LOG_DIRECTORY = "/home/y/logs/vespa/";
+    private static final String LOG_DIRECTORY = "/home/y/logs/vespa/logarchive/";
 
     @Inject
     public LogHandler(Executor executor) {
@@ -23,10 +25,15 @@ public class LogHandler extends ThreadedHttpRequestHandler {
 
     @Override
     public HttpResponse handle(HttpRequest request) {
-        JSONObject logJson;
+        JSONObject responseJSON = new JSONObject();
 
+        HashMap<String, String> apiParams = getParameters(request);
+        long earliestLogThreshold = getEarliestThreshold(apiParams);
+        long latestLogThreshold = getLatestThreshold(apiParams);
+        LogReader logReader= new LogReader(earliestLogThreshold, latestLogThreshold);
         try {
-            logJson = LogReader.readLogs(LOG_DIRECTORY);
+            JSONObject logJson = logReader.readLogs(LOG_DIRECTORY);
+            responseJSON.put("logs", logJson.toString());
         } catch (IOException | JSONException e) {
             return new HttpResponse(404) {
                 @Override
@@ -37,9 +44,33 @@ public class LogHandler extends ThreadedHttpRequestHandler {
             @Override
             public void render(OutputStream outputStream) throws IOException {
                 OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream);
-                outputStreamWriter.write(logJson.toString());
+                outputStreamWriter.write(responseJSON.toString());
                 outputStreamWriter.close();
             }
         };
     }
+
+    private HashMap<String, String> getParameters(HttpRequest request) {
+        String query = request.getUri().getQuery();
+        HashMap<String, String> keyValPair = new HashMap<>();
+        Arrays.stream(query.split("&")).forEach(pair -> {
+            String[] splitPair = pair.split("=");
+            keyValPair.put(splitPair[0], splitPair[1]);
+        });
+        return keyValPair;
+    }
+
+    private long getEarliestThreshold(HashMap<String, String> map) {
+        if (map.containsKey("from")) {
+            return Long.valueOf(map.get("from"));
+        }
+        return Long.MIN_VALUE;
+    }
+
+    private long getLatestThreshold(HashMap<String, String> map) {
+        if (map.containsKey("to")) {
+            return Long.valueOf(map.get("to"));
+        }
+        return Long.MAX_VALUE;
+    }
 }
diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
index eb00446dd0e..2483f2497d0 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java
@@ -10,23 +10,34 @@ import java.nio.file.Files;
 
 public class LogReader {
 
-    protected static JSONObject readLogs(String logDirectory) throws IOException, JSONException {
+    long earliestLogThreshold;
+    long latestLogThreshold;
+
+    public LogReader(long earliestLogThreshold, long latestLogThreshold) {
+        this.earliestLogThreshold = earliestLogThreshold;
+        this.latestLogThreshold = latestLogThreshold;
+    }
+
+    protected JSONObject readLogs(String logDirectory) throws IOException, JSONException {
         JSONObject json = new JSONObject();
         File root = new File(logDirectory);
-        traverse_folder(root, json);
+        traverse_folder(root, json, "");
         return json;
     }
 
-    private static void traverse_folder(File root, JSONObject json) throws IOException, JSONException {
-        for(File child : root.listFiles()) {
+    private void traverse_folder(File root, JSONObject json, String filename) throws IOException, JSONException {
+        File[] files = root.listFiles();
+        for(File child : files) {
+            File temp = child;
             JSONObject childJson = new JSONObject();
-            if(child.isFile()) {
-                json.put(child.getName(), DatatypeConverter.printBase64Binary(Files.readAllBytes(child.toPath())));
+            long logTime = child.lastModified();
+            if(child.isFile() && earliestLogThreshold < logTime && logTime < latestLogThreshold) {
+                json.put(filename + child.getName(), DatatypeConverter.printBase64Binary(Files.readAllBytes(child.toPath())));
             }
-            else {
-                json.put(child.getName(), childJson);
-                traverse_folder(child, childJson);
+            else if (!child.isFile()){
+                traverse_folder(child, json, filename + child.getName() + "-");
             }
         }
     }
+
 }
diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
index e5302ee43ee..534026f89ac 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
@@ -20,8 +20,19 @@ public class LogReaderTest {
     @Test
     public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{
         String logDirectory = "src/test/resources/logfolder/";
-        JSONObject json = LogReader.readLogs(logDirectory);
-        String expected = "{\"subfolder\":{\"log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"},\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}";
+        LogReader logReader = new LogReader(21, Long.MAX_VALUE);
+        JSONObject json = logReader.readLogs(logDirectory);
+        String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}";
+        String actual = json.toString();
+        assertEquals(expected, actual);
+    }
+
+    @Test
+    public void testThatLogsOutsideRangeAreExcluded() throws Exception {
+        String logDirectory = "src/test/resources/logfolder/";
+        LogReader logReader = new LogReader(Long.MAX_VALUE, Long.MIN_VALUE);
+        JSONObject json = logReader.readLogs(logDirectory);
+        String expected = "{}";
         String actual = json.toString();
         assertEquals(expected, actual);
     }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
index eb10c78f891..5dacaf9b0db 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
@@ -1,7 +1,6 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.integration.configserver;
 
-import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
@@ -9,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -42,7 +42,7 @@ public interface ConfigServer {
 
     Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath);
 
-    HttpResponse getLogs(DeploymentId deployment);
+    Optional<Logs> getLogs(DeploymentId deployment, HashMap<String, String> queryParameters);
     /**
      * Set new status on en endpoint in one zone.
      *
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java
new file mode 100644
index 00000000000..223d3e88f13
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java
@@ -0,0 +1,16 @@
+package com.yahoo.vespa.hosted.controller.api.integration.configserver;
+
+import java.util.Map;
+
+public class Logs {
+
+    private final Map<String, String> logs;
+
+    public Logs(Map<String, String> logs) {
+        this.logs = logs;
+    }
+
+    public Map<String, String> logs() {
+        return this.logs;
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 034db3d487d..154c4e632de 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -49,6 +49,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus;
@@ -87,7 +88,9 @@ import java.net.URISyntaxException;
 import java.security.Principal;
 import java.time.DayOfWeek;
 import java.time.Duration;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -168,7 +171,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request);
-        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.getUri().getQuery());
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri());
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri());
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after"));
@@ -346,13 +349,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return new SlimeJsonResponse(slime);
     }
 
-    private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region) {
+    private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region, String query) {
         ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName);
         ZoneId zone = ZoneId.from(environment, region);
         DeploymentId deployment = new DeploymentId(application, zone);
-        return controller.configServer().getLogs(deployment);
+        HashMap<String, String> queryParameters = getParameters(query);
+        Optional<Logs> response = controller.configServer().getLogs(deployment, queryParameters);
+        Slime slime = new Slime();
+        Cursor object = slime.setObject();
+        if (response.isPresent()) {
+            response.get().logs().entrySet().stream().forEach(entry -> object.setString(entry.getKey(), entry.getValue()));
+        }
+        return new SlimeJsonResponse(slime);
     }
 
+    private HashMap<String, String> getParameters(String query) {
+        HashMap<String, String> keyValPair = new HashMap<>();
+        Arrays.stream(query.split("&")).forEach(pair -> {
+            String[] splitPair = pair.split("=");
+            keyValPair.put(splitPair[0], splitPair[1]);
+        });
+        return keyValPair;
+    }
 
     private void toSlime(Cursor object, Application application, HttpRequest request) {
         object.setString("application", application.id().application().value());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index bd65465633e..aea809de365 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence;
@@ -298,14 +299,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     @Override
-    public HttpResponse getLogs(DeploymentId deployment) {
-        return new HttpResponse(200) {
-            @Override
-            public void render(OutputStream outputStream) throws IOException {
-                outputStream.write("{\"subfolder\":{\"log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"},\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}".getBytes());
-            }
-        };
-
+    public Optional<Logs> getLogs(DeploymentId deployment, HashMap<String, String> queryParameters) {
+        HashMap<String, String> logs = new HashMap<>();
+        logs.put("subfolder-log2.log", "VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl");
+        logs.put("log1.log", "VGhpcyBpcyBvbmUgbG9nIGZpbGU=");
+        return Optional.of(new Logs(logs));
     }
 
     public static class Application {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 0ea23ae1b78..30c81a0721a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -331,7 +331,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               new File("application1-recursive.json"));
 
         // GET logs
-        tester.assertResponse(request("/application/v4/tenant/tenant2/application//application1/environment/prod/region/corp-us-east-1/instance/default/logs", GET).userIdentity(USER_ID), new File("logs.json"));
+        tester.assertResponse(request("/application/v4/tenant/tenant2/application//application1/environment/prod/region/corp-us-east-1/instance/default/logs?from=1233&to=3214", GET).userIdentity(USER_ID), new File("logs.json"));
 
         // DELETE (cancel) ongoing change
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json
index 398a62758ee..69fc0f88ea6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json
@@ -1,5 +1,4 @@
 {
-  "subfolder": {
-    "log2.log":"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl"},
+  "subfolder-log2.log":"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl",
   "log1.log":"VGhpcyBpcyBvbmUgbG9nIGZpbGU="
 }
\ No newline at end of file