diff options
author | valerijf <valerijf@yahoo-inc.com> | 2017-03-08 14:18:06 +0100 |
---|---|---|
committer | valerijf <valerijf@yahoo-inc.com> | 2017-03-08 14:18:06 +0100 |
commit | 91cf8827f3d9b286e19f69677de8afce0efac4ae (patch) | |
tree | a697115a35f1770f52e202ce24b4482975734bc3 /node-maintainer | |
parent | a4faaca346729386c88484059af8b4ebd355bc5a (diff) |
Created REST API for node-maintainer
Diffstat (limited to 'node-maintainer')
3 files changed, 191 insertions, 127 deletions
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java index cd7d8834fba..dd7d6c5c93e 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java @@ -32,7 +32,7 @@ public class CoreCollector { private static final Logger logger = Logger.getLogger(CoreCollector.class.getName()); private final ProcessExecuter processExecuter; - CoreCollector(ProcessExecuter processExecuter) { + public CoreCollector(ProcessExecuter processExecuter) { this.processExecuter = processExecuter; } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java index 3d677865a3b..7a502476e0f 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java @@ -1,122 +1,32 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.maintainer; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.system.ProcessExecuter; -import org.apache.http.impl.client.HttpClientBuilder; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Type; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.node.maintainer.restapi.v1.MaintainerApiHandler; /** * @author freva */ public class Maintainer { - - @SuppressWarnings("unchecked") - public static void main(String[] args) throws Exception { + public static void main(String[] args) { if (args.length != 1) { throw new RuntimeException("Expected only 1 argument - a JSON list of maintainer jobs to execute"); } - ObjectMapper mapper = new ObjectMapper(); - List<MaintenanceJob> maintenanceJobs = mapper.readValue(args[0], new TypeReference<List<MaintenanceJob>>(){}); - executeJobs(maintenanceJobs); - } - - public static void executeJobs(List<MaintenanceJob> maintenanceJobs) { - for (MaintenanceJob job : maintenanceJobs) { - try { - executeJob(job); - } catch (Exception e) { - throw new RuntimeException("Failed to execute job " + job.jobName + " with arguments " + - Arrays.toString(job.arguments.entrySet().toArray()), e); - } + MaintainerApiHandler handler = new MaintainerApiHandler(Runnable::run, AccessLog.voidAccessLog()); + Inspector object = SlimeUtils.jsonToSlime(args[0].getBytes()).get(); + if (object.type() != Type.ARRAY) { + throw new IllegalArgumentException("Expected a list maintainer jobs to execute"); } - } - - @SuppressWarnings("unchecked") - private static void executeJob(MaintenanceJob job) throws IOException { - switch (job.getJobName()) { - case "delete-files": - DeleteOldAppData.deleteFiles( - (String) job.getRequiredArgument("basePath"), - (Integer) job.getRequiredArgument("maxAgeSeconds"), - (String) job.getArgumentOrDefault("fileNameRegex", null), - (boolean) job.getArgumentOrDefault("recursive", false)); - break; - - case "delete-directories": - DeleteOldAppData.deleteDirectories( - (String) job.getRequiredArgument("basePath"), - (Integer) job.getRequiredArgument("maxAgeSeconds"), - (String) job.getArgumentOrDefault("dirNameRegex", null)); - break; - - case "recursive-delete": - DeleteOldAppData.recursiveDelete( - (String) job.getRequiredArgument("path")); - break; - - case "move-files": - Path from = Paths.get((String) job.getRequiredArgument("from")); - Path to = Paths.get((String) job.getRequiredArgument("to")); - if (Files.exists(from)) { - Files.move(from, to); - } - break; - - case "handle-core-dumps": - CoreCollector coreCollector = new CoreCollector(new ProcessExecuter()); - CoredumpHandler coredumpHandler = new CoredumpHandler(HttpClientBuilder.create().build(), coreCollector); - - Path containerCoredumpsPath = Paths.get((String) job.getRequiredArgument("containerCoredumpsPath")); - Path doneCoredumpsPath = Paths.get((String) job.getRequiredArgument("doneCoredumpsPath")); - Map<String, Object> attributesMap = (Map<String, Object>) job.getRequiredArgument("attributes"); - - coredumpHandler.removeJavaCoredumps(containerCoredumpsPath); - coredumpHandler.processAndReportCoredumps(containerCoredumpsPath, doneCoredumpsPath, attributesMap); - break; - default: - throw new RuntimeException("Unknown job: " + job.getJobName()); - } - } - - /** - * Should be equal to MaintainerExecutorJob in StorageMaintainer - */ - public static class MaintenanceJob { - private final String jobName; - private final Map<String, Object> arguments; - - private MaintenanceJob(@JsonProperty(value="jobName") String jobName, - @JsonProperty(value="arguments") Map<String, Object> arguments) { - this.jobName = jobName; - this.arguments = arguments; - } - - String getJobName() { - return jobName; - } - - Object getRequiredArgument(String argumentName) { - Object value = arguments.get(argumentName); - if (value == null) { - throw new IllegalArgumentException("Missing required argument " + argumentName); - } - return value; - } - - Object getArgumentOrDefault(String argumentName, Object defaultValue) { - return arguments.getOrDefault(argumentName, defaultValue); - } + object.traverse((ArrayTraverser) (int i, Inspector item) -> { + String type = handler.getFieldOrFail(item, "type").asString(); + Inspector arguments = handler.getFieldOrFail(item, "arguments"); + handler.parseMaintenanceJob(type, arguments); + }); } } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/restapi/v1/MaintainerApiHandler.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/restapi/v1/MaintainerApiHandler.java index b8f97b0bb3f..8564746a27d 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/restapi/v1/MaintainerApiHandler.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/restapi/v1/MaintainerApiHandler.java @@ -1,17 +1,31 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.maintainer.restapi.v1; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.logging.AccessLog; -import com.yahoo.vespa.hosted.node.maintainer.Maintainer; +import com.yahoo.io.IOUtils; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.slime.Type; +import com.yahoo.system.ProcessExecuter; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.node.maintainer.CoreCollector; +import com.yahoo.vespa.hosted.node.maintainer.CoredumpHandler; +import com.yahoo.vespa.hosted.node.maintainer.DeleteOldAppData; import com.yahoo.yolean.Exceptions; +import org.apache.http.impl.client.HttpClientBuilder; import java.io.IOException; -import java.util.List; +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Level; @@ -20,7 +34,8 @@ import java.util.logging.Level; * @author freva */ public class MaintainerApiHandler extends LoggingRequestHandler { - private final static ObjectMapper objectMapper = new ObjectMapper(); + private static final CoreCollector coreCollector = new CoreCollector(new ProcessExecuter()); + private static final CoredumpHandler coredumpHandler = new CoredumpHandler(HttpClientBuilder.create().build(), coreCollector); public MaintainerApiHandler(Executor executor, AccessLog accessLog) { super(executor, accessLog); @@ -34,9 +49,6 @@ public class MaintainerApiHandler extends LoggingRequestHandler { default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } } -// catch (NotFoundException e) { -// return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); -// } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } @@ -47,19 +59,161 @@ public class MaintainerApiHandler extends LoggingRequestHandler { } private HttpResponse handlePOST(HttpRequest request) { - switch (request.getUri().getPath()) { - case "/maintainer/v1": - try { - List<Maintainer.MaintenanceJob> maintenanceJobs = objectMapper.readValue( - request.getData(), new TypeReference<List<Maintainer.MaintenanceJob>>(){}); - Maintainer.executeJobs(maintenanceJobs); - } catch (IOException e) { - throw new RuntimeException("Failed parsing JSON request", e); - } - return new MessageResponse("Successfully executed command"); + String requestPath = request.getUri().getPath(); + if (requestPath.equals("/maintainer/v1")) { + Inspector object = toSlime(request.getData()).get(); + if (object.type() != Type.ARRAY) { + throw new IllegalArgumentException("Expected a list maintainer jobs to execute"); + } + + object.traverse((ArrayTraverser) (int i, Inspector item) -> { + String type = getFieldOrFail(item, "type").asString(); + Inspector arguments = getFieldOrFail(item, "arguments"); + parseMaintenanceJob(type, arguments); + }); + return new MessageResponse("Successfully executed " + object.entries() + " commands"); + + } else if (requestPath.startsWith("/maintainer/v1/")) { + String type = requestPath.substring(requestPath.lastIndexOf('/') + 1); + Inspector arguments = toSlime(request.getData()).get(); + parseMaintenanceJob(type, arguments); + return new MessageResponse("Successfully executed " + type); + + } else { + return ErrorResponse.notFoundError("Nothing at path '" + requestPath + "'"); + } + } + + public void parseMaintenanceJob(String type, Inspector arguments) { + if (arguments.type() != Type.OBJECT) { + throw new IllegalArgumentException("Expected a 'arguments' to be an object"); + } + + switch (type) { + case "delete-files": + parseDeleteFilesJob(arguments); + break; + + case "delete-directories": + parseDeleteDirectoriesJob(arguments); + break; + + case "recursive-delete": + parseRecursiveDelete(arguments); + break; + + case "move-files": + parseMoveFiles(arguments); + break; + + case "handle-core-dumps": + parseHandleCoreDumps(arguments); + break; + default: - return ErrorResponse.notFoundError("Fail"); -// throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); + throw new IllegalArgumentException("Unknown job: " + type); + } + } + + private void parseDeleteFilesJob(Inspector arguments) { + Path basePath = Paths.get(getFieldOrFail(arguments, "basePath").asString()); + Duration maxAge = Duration.ofSeconds(getFieldOrFail(arguments, "maxAgeSeconds").asLong()); + Optional<String> fileNameRegex = SlimeUtils.optionalString(getFieldOrFail(arguments, "fileNameRegex")); + boolean recursive = getFieldOrFail(arguments, "recursive").asBool(); + try { + DeleteOldAppData.deleteFiles(basePath, maxAge, fileNameRegex, recursive); + } catch (IOException e) { + throw new RuntimeException("Failed deleting files under " + basePath.toAbsolutePath() + + fileNameRegex.map(regex -> ", matching '" + regex + "'").orElse("") + + ", " + (recursive ? "" : "not ") + "recursively" + + " and older than " + maxAge, e); + } + } + + private void parseDeleteDirectoriesJob(Inspector arguments) { + Path basePath = Paths.get(getFieldOrFail(arguments, "basePath").asString()); + Duration maxAge = Duration.ofSeconds(getFieldOrFail(arguments, "maxAgeSeconds").asLong()); + Optional<String> dirNameRegex = SlimeUtils.optionalString(getFieldOrFail(arguments, "dirNameRegex")); + try { + DeleteOldAppData.deleteDirectories(basePath, maxAge, dirNameRegex); + } catch (IOException e) { + throw new RuntimeException("Failed deleting directories under " + basePath.toAbsolutePath() + + dirNameRegex.map(regex -> ", matching '" + regex + "'").orElse("") + + " and older than " + maxAge, e); + } + } + + private void parseRecursiveDelete(Inspector arguments) { + Path basePath = Paths.get(getFieldOrFail(arguments, "path").asString()); + try { + DeleteOldAppData.recursiveDelete(basePath); + } catch (IOException e) { + throw new RuntimeException("Failed deleting " + basePath.toAbsolutePath(), e); + } + } + + private void parseMoveFiles(Inspector arguments) { + Path from = Paths.get(getFieldOrFail(arguments, "from").asString()); + Path to = Paths.get(getFieldOrFail(arguments, "to").asString()); + + try { + DeleteOldAppData.moveIfExists(from, to); + } catch (IOException e) { + throw new RuntimeException("Failed moving from " + from.toAbsolutePath() + ", to " + to.toAbsolutePath(), e); + } + } + + private void parseHandleCoreDumps(Inspector arguments) { + Path coredumpsPath = Paths.get(getFieldOrFail(arguments, "coredumpsPath").asString()); + Path doneCoredumpsPath = Paths.get(getFieldOrFail(arguments, "doneCoredumpsPath").asString()); + Map<String, Object> attributesMap = parseMap(getFieldOrFail(arguments, "attributes")); + + try { + coredumpHandler.removeJavaCoredumps(coredumpsPath); + coredumpHandler.processAndReportCoredumps(coredumpsPath, doneCoredumpsPath, attributesMap); + } catch (IOException e) { + throw new RuntimeException("Failed processing coredumps at " + coredumpsPath.toAbsolutePath() + + ", moving fished dumps to " + doneCoredumpsPath.toAbsolutePath(), e); + } + } + + private Map<String, Object> parseMap(Inspector object) { + Map<String, Object> map = new HashMap<>(); + getFieldOrFail(object, "attributes").traverse((String key, Inspector value) -> { + switch (value.type()) { + case BOOL: + map.put(key, value.asBool()); + break; + case LONG: + map.put(key, value.asLong()); + break; + case DOUBLE: + map.put(key, value.asDouble()); + break; + case STRING: + map.put(key, value.asString()); + break; + default: + throw new IllegalArgumentException("Invalid attribute for key '" + key + "', value " + value); + } + }); + return map; + } + + private Slime toSlime(InputStream jsonStream) { + try { + byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); + return SlimeUtils.jsonToSlime(jsonBytes); + } catch (IOException e) { + throw new RuntimeException(); + } + } + + public Inspector getFieldOrFail(Inspector object, String key) { + Inspector out = object.field(key); + if (out.type() == Type.NIX) { + throw new IllegalArgumentException("Key '" + key + "' was not found!"); } + return out; } } |