Created REST API for node-maintainer

3 files changed, 191 insertions, 127 deletions

diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java
index cd7d8834fba..dd7d6c5c93e 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java
@@ -32,7 +32,7 @@ public class CoreCollector {
     private static final Logger logger = Logger.getLogger(CoreCollector.class.getName());
     private final ProcessExecuter processExecuter;
 
-    CoreCollector(ProcessExecuter processExecuter) {
+    public CoreCollector(ProcessExecuter processExecuter) {
         this.processExecuter = processExecuter;
     }
     
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java
index 3d677865a3b..7a502476e0f 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java
@@ -1,122 +1,32 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.maintainer;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.yahoo.system.ProcessExecuter;
-import org.apache.http.impl.client.HttpClientBuilder;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Type;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.node.maintainer.restapi.v1.MaintainerApiHandler;
 
 /**
  * @author freva
  */
 public class Maintainer {
-
-    @SuppressWarnings("unchecked")
-    public static void main(String[] args) throws Exception {
+    public static void main(String[] args) {
         if (args.length != 1) {
             throw new RuntimeException("Expected only 1 argument - a JSON list of maintainer jobs to execute");
         }
 
-        ObjectMapper mapper = new ObjectMapper();
-        List<MaintenanceJob> maintenanceJobs = mapper.readValue(args[0], new TypeReference<List<MaintenanceJob>>(){});
-        executeJobs(maintenanceJobs);
-    }
-
-    public static void executeJobs(List<MaintenanceJob> maintenanceJobs) {
-        for (MaintenanceJob job : maintenanceJobs) {
-            try {
-                executeJob(job);
-            } catch (Exception e) {
-                throw new RuntimeException("Failed to execute job " + job.jobName + " with arguments " +
-                        Arrays.toString(job.arguments.entrySet().toArray()), e);
-            }
+        MaintainerApiHandler handler = new MaintainerApiHandler(Runnable::run, AccessLog.voidAccessLog());
+        Inspector object = SlimeUtils.jsonToSlime(args[0].getBytes()).get();
+        if (object.type() != Type.ARRAY) {
+            throw new IllegalArgumentException("Expected a list maintainer jobs to execute");
         }
-    }
-
-    @SuppressWarnings("unchecked")
-    private static void executeJob(MaintenanceJob job) throws IOException {
-        switch (job.getJobName()) {
-            case "delete-files":
-                DeleteOldAppData.deleteFiles(
-                        (String) job.getRequiredArgument("basePath"),
-                        (Integer) job.getRequiredArgument("maxAgeSeconds"),
-                        (String) job.getArgumentOrDefault("fileNameRegex", null),
-                        (boolean) job.getArgumentOrDefault("recursive", false));
-                break;
-
-            case "delete-directories":
-                DeleteOldAppData.deleteDirectories(
-                        (String) job.getRequiredArgument("basePath"),
-                        (Integer) job.getRequiredArgument("maxAgeSeconds"),
-                        (String) job.getArgumentOrDefault("dirNameRegex", null));
-                break;
-
-            case "recursive-delete":
-                DeleteOldAppData.recursiveDelete(
-                        (String) job.getRequiredArgument("path"));
-                break;
-
-            case "move-files":
-                Path from = Paths.get((String) job.getRequiredArgument("from"));
-                Path to = Paths.get((String) job.getRequiredArgument("to"));
-                if (Files.exists(from)) {
-                    Files.move(from, to);
-                }
-                break;
-
-            case "handle-core-dumps":
-                CoreCollector coreCollector = new CoreCollector(new ProcessExecuter());
-                CoredumpHandler coredumpHandler = new CoredumpHandler(HttpClientBuilder.create().build(), coreCollector);
-
-                Path containerCoredumpsPath = Paths.get((String) job.getRequiredArgument("containerCoredumpsPath"));
-                Path doneCoredumpsPath = Paths.get((String) job.getRequiredArgument("doneCoredumpsPath"));
-                Map<String, Object> attributesMap = (Map<String, Object>) job.getRequiredArgument("attributes");
-
-                coredumpHandler.removeJavaCoredumps(containerCoredumpsPath);
-                coredumpHandler.processAndReportCoredumps(containerCoredumpsPath, doneCoredumpsPath, attributesMap);
-                break;
 
-            default:
-                throw new RuntimeException("Unknown job: " + job.getJobName());
-        }
-    }
-
-    /**
-     * Should be equal to MaintainerExecutorJob in StorageMaintainer
-     */
-    public static class MaintenanceJob {
-        private final String jobName;
-        private final Map<String, Object> arguments;
-
-        private MaintenanceJob(@JsonProperty(value="jobName") String jobName,
-                               @JsonProperty(value="arguments") Map<String, Object> arguments) {
-            this.jobName = jobName;
-            this.arguments = arguments;
-        }
-
-        String getJobName() {
-            return jobName;
-        }
-
-        Object getRequiredArgument(String argumentName) {
-            Object value = arguments.get(argumentName);
-            if (value == null) {
-                throw new IllegalArgumentException("Missing required argument " + argumentName);
-            }
-            return value;
-        }
-
-        Object getArgumentOrDefault(String argumentName, Object defaultValue) {
-            return arguments.getOrDefault(argumentName, defaultValue);
-        }
+        object.traverse((ArrayTraverser) (int i, Inspector item) -> {
+            String type = handler.getFieldOrFail(item, "type").asString();
+            Inspector arguments = handler.getFieldOrFail(item, "arguments");
+            handler.parseMaintenanceJob(type, arguments);
+        });
     }
 }
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/restapi/v1/MaintainerApiHandler.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/restapi/v1/MaintainerApiHandler.java
index b8f97b0bb3f..8564746a27d 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/restapi/v1/MaintainerApiHandler.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/restapi/v1/MaintainerApiHandler.java
@@ -1,17 +1,31 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.maintainer.restapi.v1;
 
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.container.logging.AccessLog;
-import com.yahoo.vespa.hosted.node.maintainer.Maintainer;
+import com.yahoo.io.IOUtils;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.Type;
+import com.yahoo.system.ProcessExecuter;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.node.maintainer.CoreCollector;
+import com.yahoo.vespa.hosted.node.maintainer.CoredumpHandler;
+import com.yahoo.vespa.hosted.node.maintainer.DeleteOldAppData;
 import com.yahoo.yolean.Exceptions;
+import org.apache.http.impl.client.HttpClientBuilder;
 
 import java.io.IOException;
-import java.util.List;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.Executor;
 import java.util.logging.Level;
 
@@ -20,7 +34,8 @@ import java.util.logging.Level;
  * @author freva
  */
 public class MaintainerApiHandler extends LoggingRequestHandler {
-    private final static ObjectMapper objectMapper = new ObjectMapper();
+    private static final CoreCollector coreCollector = new CoreCollector(new ProcessExecuter());
+    private static final CoredumpHandler coredumpHandler = new CoredumpHandler(HttpClientBuilder.create().build(), coreCollector);
 
     public MaintainerApiHandler(Executor executor, AccessLog accessLog) {
         super(executor, accessLog);
@@ -34,9 +49,6 @@ public class MaintainerApiHandler extends LoggingRequestHandler {
                 default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
             }
         }
-//        catch (NotFoundException e) {
-//            return ErrorResponse.notFoundError(Exceptions.toMessageString(e));
-//        }
         catch (IllegalArgumentException e) {
             return ErrorResponse.badRequest(Exceptions.toMessageString(e));
         }
@@ -47,19 +59,161 @@ public class MaintainerApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse handlePOST(HttpRequest request) {
-        switch (request.getUri().getPath()) {
-            case "/maintainer/v1":
-                try {
-                    List<Maintainer.MaintenanceJob> maintenanceJobs = objectMapper.readValue(
-                            request.getData(), new TypeReference<List<Maintainer.MaintenanceJob>>(){});
-                    Maintainer.executeJobs(maintenanceJobs);
-                } catch (IOException e) {
-                    throw new RuntimeException("Failed parsing JSON request", e);
-                }
-                return new MessageResponse("Successfully executed command");
+        String requestPath = request.getUri().getPath();
+        if (requestPath.equals("/maintainer/v1")) {
+            Inspector object = toSlime(request.getData()).get();
+            if (object.type() != Type.ARRAY) {
+                throw new IllegalArgumentException("Expected a list maintainer jobs to execute");
+            }
+
+            object.traverse((ArrayTraverser) (int i, Inspector item) -> {
+                String type = getFieldOrFail(item, "type").asString();
+                Inspector arguments = getFieldOrFail(item, "arguments");
+                parseMaintenanceJob(type, arguments);
+            });
+            return new MessageResponse("Successfully executed " + object.entries() + " commands");
+
+        } else if (requestPath.startsWith("/maintainer/v1/")) {
+            String type = requestPath.substring(requestPath.lastIndexOf('/') + 1);
+            Inspector arguments = toSlime(request.getData()).get();
+            parseMaintenanceJob(type, arguments);
+            return new MessageResponse("Successfully executed " + type);
+
+        } else {
+            return ErrorResponse.notFoundError("Nothing at path '" + requestPath + "'");
+        }
+    }
+
+    public void parseMaintenanceJob(String type, Inspector arguments) {
+        if (arguments.type() != Type.OBJECT) {
+            throw new IllegalArgumentException("Expected a 'arguments' to be an object");
+        }
+
+        switch (type) {
+            case "delete-files":
+                parseDeleteFilesJob(arguments);
+                break;
+
+            case "delete-directories":
+                parseDeleteDirectoriesJob(arguments);
+                break;
+
+            case "recursive-delete":
+                parseRecursiveDelete(arguments);
+                break;
+
+            case "move-files":
+                parseMoveFiles(arguments);
+                break;
+
+            case "handle-core-dumps":
+                parseHandleCoreDumps(arguments);
+                break;
+
             default:
-                return ErrorResponse.notFoundError("Fail");
-//                throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'");
+                throw new IllegalArgumentException("Unknown job: " + type);
+        }
+    }
+
+    private void parseDeleteFilesJob(Inspector arguments) {
+        Path basePath = Paths.get(getFieldOrFail(arguments, "basePath").asString());
+        Duration maxAge = Duration.ofSeconds(getFieldOrFail(arguments, "maxAgeSeconds").asLong());
+        Optional<String> fileNameRegex = SlimeUtils.optionalString(getFieldOrFail(arguments, "fileNameRegex"));
+        boolean recursive = getFieldOrFail(arguments, "recursive").asBool();
+        try {
+            DeleteOldAppData.deleteFiles(basePath, maxAge, fileNameRegex, recursive);
+        } catch (IOException e) {
+            throw new RuntimeException("Failed deleting files under " + basePath.toAbsolutePath() +
+                    fileNameRegex.map(regex -> ", matching '" + regex + "'").orElse("") +
+                    ", " + (recursive ? "" : "not ") + "recursively" +
+                    " and older than " + maxAge, e);
+        }
+    }
+
+    private void parseDeleteDirectoriesJob(Inspector arguments) {
+        Path basePath = Paths.get(getFieldOrFail(arguments, "basePath").asString());
+        Duration maxAge = Duration.ofSeconds(getFieldOrFail(arguments, "maxAgeSeconds").asLong());
+        Optional<String> dirNameRegex = SlimeUtils.optionalString(getFieldOrFail(arguments, "dirNameRegex"));
+        try {
+            DeleteOldAppData.deleteDirectories(basePath, maxAge, dirNameRegex);
+        } catch (IOException e) {
+            throw new RuntimeException("Failed deleting directories under " + basePath.toAbsolutePath() +
+                    dirNameRegex.map(regex -> ", matching '" + regex + "'").orElse("") +
+                    " and older than " + maxAge, e);
+        }
+    }
+
+    private void parseRecursiveDelete(Inspector arguments) {
+        Path basePath = Paths.get(getFieldOrFail(arguments, "path").asString());
+        try {
+            DeleteOldAppData.recursiveDelete(basePath);
+        } catch (IOException e) {
+            throw new RuntimeException("Failed deleting " + basePath.toAbsolutePath(), e);
+        }
+    }
+
+    private void parseMoveFiles(Inspector arguments) {
+        Path from = Paths.get(getFieldOrFail(arguments, "from").asString());
+        Path to = Paths.get(getFieldOrFail(arguments, "to").asString());
+
+        try {
+            DeleteOldAppData.moveIfExists(from, to);
+        } catch (IOException e) {
+            throw new RuntimeException("Failed moving from " + from.toAbsolutePath() + ", to " + to.toAbsolutePath(), e);
+        }
+    }
+
+    private void parseHandleCoreDumps(Inspector arguments) {
+        Path coredumpsPath = Paths.get(getFieldOrFail(arguments, "coredumpsPath").asString());
+        Path doneCoredumpsPath = Paths.get(getFieldOrFail(arguments, "doneCoredumpsPath").asString());
+        Map<String, Object> attributesMap = parseMap(getFieldOrFail(arguments, "attributes"));
+
+        try {
+            coredumpHandler.removeJavaCoredumps(coredumpsPath);
+            coredumpHandler.processAndReportCoredumps(coredumpsPath, doneCoredumpsPath, attributesMap);
+        } catch (IOException e) {
+            throw new RuntimeException("Failed processing coredumps at " + coredumpsPath.toAbsolutePath() +
+                    ", moving fished dumps to " + doneCoredumpsPath.toAbsolutePath(), e);
+        }
+    }
+
+    private Map<String, Object> parseMap(Inspector object) {
+        Map<String, Object> map = new HashMap<>();
+        getFieldOrFail(object, "attributes").traverse((String key, Inspector value) -> {
+            switch (value.type()) {
+                case BOOL:
+                    map.put(key, value.asBool());
+                    break;
+                case LONG:
+                    map.put(key, value.asLong());
+                    break;
+                case DOUBLE:
+                    map.put(key, value.asDouble());
+                    break;
+                case STRING:
+                    map.put(key, value.asString());
+                    break;
+                default:
+                    throw new IllegalArgumentException("Invalid attribute for key '" + key + "', value " + value);
+            }
+        });
+        return map;
+    }
+
+    private Slime toSlime(InputStream jsonStream) {
+        try {
+            byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000);
+            return SlimeUtils.jsonToSlime(jsonBytes);
+        } catch (IOException e) {
+            throw new RuntimeException();
+        }
+    }
+
+    public Inspector getFieldOrFail(Inspector object, String key) {
+        Inspector out = object.field(key);
+        if (out.type() == Type.NIX) {
+            throw new IllegalArgumentException("Key '" + key + "' was not found!");
         }
+        return out;
     }
 }