diff options
12 files changed, 125 insertions, 39 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 6a55fb77933..6c67f730f60 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -483,10 +483,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Logs ---------------------------------------------------------------- - public HttpResponse getLogs(ApplicationId applicationId) { - String logServerHostName = getLogServerURI(applicationId); + public HttpResponse getLogs(ApplicationId applicationId, String apiParams) { + String logServerURI = getLogServerURI(applicationId) + apiParams; LogRetriever logRetriever = new LogRetriever(); - return logRetriever.getLogs(logServerHostName); + return logRetriever.getLogs(logServerURI); } // ---------------- Session operations ---------------------------------------------------------------- diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index b65cb370f93..528575f4f27 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -97,7 +97,9 @@ public class ApplicationHandler extends HttpHandler { } if (isLogRequest(request)) { - return applicationRepository.getLogs(applicationId); + String apiParams = request.getUri().getQuery(); + apiParams = apiParams == null ? "" : "?" + apiParams; + return applicationRepository.getLogs(applicationId, apiParams); } return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 120119f35bb..2ae8917e905 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -118,7 +118,7 @@ public class ApplicationRepositoryTest { } @Test - public void getLogs(){ + public void getLogs() { WireMockServer wireMock = new WireMockServer(wireMockConfig().port(8080)); wireMock.start(); WireMock.configureFor("localhost", wireMock.port()); @@ -127,15 +127,15 @@ public class ApplicationRepositoryTest { .withStatus(200))); wireMock.start(); deployApp(testAppLogServerWithContainer); - HttpResponse response = applicationRepository.getLogs(applicationId()); - assertEquals(response.getStatus(),200); + HttpResponse response = applicationRepository.getLogs(applicationId(), ""); + assertEquals(200, response.getStatus()); wireMock.stop(); } @Test(expected = IllegalArgumentException.class) public void getLogsNoContainerOnLogServerHostShouldThrowException() { deployApp(testApp); - applicationRepository.getLogs(applicationId()); + applicationRepository.getLogs(applicationId(), ""); } @Test diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 4183b642af1..4c12bacf145 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -10,11 +10,13 @@ import org.json.JSONObject; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.util.Arrays; +import java.util.HashMap; import java.util.concurrent.Executor; public class LogHandler extends ThreadedHttpRequestHandler { - private static final String LOG_DIRECTORY = "/home/y/logs/vespa/"; + private static final String LOG_DIRECTORY = "/home/y/logs/vespa/logarchive/"; @Inject public LogHandler(Executor executor) { @@ -23,10 +25,15 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public HttpResponse handle(HttpRequest request) { - JSONObject logJson; + JSONObject responseJSON = new JSONObject(); + HashMap<String, String> apiParams = getParameters(request); + long earliestLogThreshold = getEarliestThreshold(apiParams); + long latestLogThreshold = getLatestThreshold(apiParams); + LogReader logReader= new LogReader(earliestLogThreshold, latestLogThreshold); try { - logJson = LogReader.readLogs(LOG_DIRECTORY); + JSONObject logJson = logReader.readLogs(LOG_DIRECTORY); + responseJSON.put("logs", logJson.toString()); } catch (IOException | JSONException e) { return new HttpResponse(404) { @Override @@ -37,9 +44,33 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public void render(OutputStream outputStream) throws IOException { OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream); - outputStreamWriter.write(logJson.toString()); + outputStreamWriter.write(responseJSON.toString()); outputStreamWriter.close(); } }; } + + private HashMap<String, String> getParameters(HttpRequest request) { + String query = request.getUri().getQuery(); + HashMap<String, String> keyValPair = new HashMap<>(); + Arrays.stream(query.split("&")).forEach(pair -> { + String[] splitPair = pair.split("="); + keyValPair.put(splitPair[0], splitPair[1]); + }); + return keyValPair; + } + + private long getEarliestThreshold(HashMap<String, String> map) { + if (map.containsKey("from")) { + return Long.valueOf(map.get("from")); + } + return Long.MIN_VALUE; + } + + private long getLatestThreshold(HashMap<String, String> map) { + if (map.containsKey("to")) { + return Long.valueOf(map.get("to")); + } + return Long.MAX_VALUE; + } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index eb00446dd0e..2483f2497d0 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -10,23 +10,34 @@ import java.nio.file.Files; public class LogReader { - protected static JSONObject readLogs(String logDirectory) throws IOException, JSONException { + long earliestLogThreshold; + long latestLogThreshold; + + public LogReader(long earliestLogThreshold, long latestLogThreshold) { + this.earliestLogThreshold = earliestLogThreshold; + this.latestLogThreshold = latestLogThreshold; + } + + protected JSONObject readLogs(String logDirectory) throws IOException, JSONException { JSONObject json = new JSONObject(); File root = new File(logDirectory); - traverse_folder(root, json); + traverse_folder(root, json, ""); return json; } - private static void traverse_folder(File root, JSONObject json) throws IOException, JSONException { - for(File child : root.listFiles()) { + private void traverse_folder(File root, JSONObject json, String filename) throws IOException, JSONException { + File[] files = root.listFiles(); + for(File child : files) { + File temp = child; JSONObject childJson = new JSONObject(); - if(child.isFile()) { - json.put(child.getName(), DatatypeConverter.printBase64Binary(Files.readAllBytes(child.toPath()))); + long logTime = child.lastModified(); + if(child.isFile() && earliestLogThreshold < logTime && logTime < latestLogThreshold) { + json.put(filename + child.getName(), DatatypeConverter.printBase64Binary(Files.readAllBytes(child.toPath()))); } - else { - json.put(child.getName(), childJson); - traverse_folder(child, childJson); + else if (!child.isFile()){ + traverse_folder(child, json, filename + child.getName() + "-"); } } } + } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index e5302ee43ee..534026f89ac 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -20,8 +20,19 @@ public class LogReaderTest { @Test public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{ String logDirectory = "src/test/resources/logfolder/"; - JSONObject json = LogReader.readLogs(logDirectory); - String expected = "{\"subfolder\":{\"log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"},\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}"; + LogReader logReader = new LogReader(21, Long.MAX_VALUE); + JSONObject json = logReader.readLogs(logDirectory); + String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}"; + String actual = json.toString(); + assertEquals(expected, actual); + } + + @Test + public void testThatLogsOutsideRangeAreExcluded() throws Exception { + String logDirectory = "src/test/resources/logfolder/"; + LogReader logReader = new LogReader(Long.MAX_VALUE, Long.MIN_VALUE); + JSONObject json = logReader.readLogs(logDirectory); + String expected = "{}"; String actual = json.toString(); assertEquals(expected, actual); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index eb10c78f891..5dacaf9b0db 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; -import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -9,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -42,7 +42,7 @@ public interface ConfigServer { Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath); - HttpResponse getLogs(DeploymentId deployment); + Optional<Logs> getLogs(DeploymentId deployment, HashMap<String, String> queryParameters); /** * Set new status on en endpoint in one zone. * diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java new file mode 100644 index 00000000000..223d3e88f13 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java @@ -0,0 +1,16 @@ +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import java.util.Map; + +public class Logs { + + private final Map<String, String> logs; + + public Logs(Map<String, String> logs) { + this.logs = logs; + } + + public Map<String, String> logs() { + return this.logs; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 034db3d487d..154c4e632de 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -49,6 +49,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; @@ -87,7 +88,9 @@ import java.net.URISyntaxException; import java.security.Principal; import java.time.DayOfWeek; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -168,7 +171,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.getUri().getQuery()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); @@ -346,13 +349,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region) { + private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region, String query) { ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName); ZoneId zone = ZoneId.from(environment, region); DeploymentId deployment = new DeploymentId(application, zone); - return controller.configServer().getLogs(deployment); + HashMap<String, String> queryParameters = getParameters(query); + Optional<Logs> response = controller.configServer().getLogs(deployment, queryParameters); + Slime slime = new Slime(); + Cursor object = slime.setObject(); + if (response.isPresent()) { + response.get().logs().entrySet().stream().forEach(entry -> object.setString(entry.getKey(), entry.getValue())); + } + return new SlimeJsonResponse(slime); } + private HashMap<String, String> getParameters(String query) { + HashMap<String, String> keyValPair = new HashMap<>(); + Arrays.stream(query.split("&")).forEach(pair -> { + String[] splitPair = pair.split("="); + keyValPair.put(splitPair[0], splitPair[1]); + }); + return keyValPair; + } private void toSlime(Cursor object, Application application, HttpRequest request) { object.setString("application", application.id().application().value()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index bd65465633e..aea809de365 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; @@ -298,14 +299,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public HttpResponse getLogs(DeploymentId deployment) { - return new HttpResponse(200) { - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("{\"subfolder\":{\"log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"},\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}".getBytes()); - } - }; - + public Optional<Logs> getLogs(DeploymentId deployment, HashMap<String, String> queryParameters) { + HashMap<String, String> logs = new HashMap<>(); + logs.put("subfolder-log2.log", "VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl"); + logs.put("log1.log", "VGhpcyBpcyBvbmUgbG9nIGZpbGU="); + return Optional.of(new Logs(logs)); } public static class Application { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 0ea23ae1b78..30c81a0721a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -331,7 +331,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("application1-recursive.json")); // GET logs - tester.assertResponse(request("/application/v4/tenant/tenant2/application//application1/environment/prod/region/corp-us-east-1/instance/default/logs", GET).userIdentity(USER_ID), new File("logs.json")); + tester.assertResponse(request("/application/v4/tenant/tenant2/application//application1/environment/prod/region/corp-us-east-1/instance/default/logs?from=1233&to=3214", GET).userIdentity(USER_ID), new File("logs.json")); // DELETE (cancel) ongoing change tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json index 398a62758ee..69fc0f88ea6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json @@ -1,5 +1,4 @@ { - "subfolder": { - "log2.log":"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl"}, + "subfolder-log2.log":"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl", "log1.log":"VGhpcyBpcyBvbmUgbG9nIGZpbGU=" }
\ No newline at end of file |