YUM install

9 files changed, 348 insertions, 0 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java
index 9def627e87f..226d5129cf6 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java
@@ -1,6 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.component;
 
+import com.yahoo.vespa.hosted.node.admin.task.util.process.ProcessApi;
+
 import java.nio.file.FileSystem;
 import java.util.EnumSet;
 import java.util.logging.Logger;
@@ -16,6 +18,7 @@ public interface TaskContext {
     }
 
     FileSystem fileSystem();
+    default ProcessApi processApi() { return null; }
 
     void logSystemModification(Logger logger, String actionDescription);
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess.java
new file mode 100644
index 00000000000..9c3333d41ae
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess.java
@@ -0,0 +1,17 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.nio.file.Path;
+
+public interface ChildProcess extends AutoCloseable {
+    ChildProcess waitForTermination();
+    int exitValue();
+    ChildProcess throwIfFailed();
+    String getUtf8Output();
+
+    @Override
+    void close();
+
+    // For testing only
+    Path getProcessOutputPath();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessImpl.java
new file mode 100644
index 00000000000..b9e075dfab7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessImpl.java
@@ -0,0 +1,78 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+
+import java.nio.file.Path;
+
+/**
+ * Represents a forked child process that still exists or has terminated.
+ */
+public class ChildProcessImpl implements ChildProcess {
+    private final Process process;
+    private final Path processOutputPath;
+    private final String commandLine;
+
+    ChildProcessImpl(Process process, Path processOutputPath, String commandLine) {
+        this.process = process;
+        this.processOutputPath = processOutputPath;
+        this.commandLine = commandLine;
+    }
+
+    public String getUtf8Output() {
+        return new UnixPath(processOutputPath).readUtf8File();
+    }
+
+    public ChildProcessImpl waitForTermination() {
+        while (true) {
+            try {
+                process.waitFor();
+            } catch (InterruptedException e) {
+                // ignoring
+                continue;
+            }
+
+            return this;
+        }
+    }
+
+    public int exitValue() {
+        return process.exitValue();
+    }
+
+    public ChildProcess throwIfFailed() {
+        if (process.exitValue() != 0) {
+            throw new CommandException("Execution of program [" + commandLine +
+                    "] failed, stdout/stderr was: <" + suffixOfOutputForLog() + ">");
+        }
+
+        return this;
+    }
+
+    private String suffixOfOutputForLog() {
+        String output = getUtf8Output();
+
+        final int maxTrailingChars = 300;
+        if (output.length() <= maxTrailingChars) {
+            return output;
+        }
+
+        int numSkippedChars = output.length() - maxTrailingChars;
+        output = output.substring(numSkippedChars);
+        return "[" + numSkippedChars + " chars omitted]..." + output;
+    }
+
+    @Override
+    public void close() {
+        if (process.isAlive()) {
+            process.destroyForcibly();
+            waitForTermination();
+        }
+        processOutputPath.toFile().delete();
+    }
+
+    @Override
+    public Path getProcessOutputPath() {
+        return processOutputPath;
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Command.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Command.java
new file mode 100644
index 00000000000..72aa5b0df2d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Command.java
@@ -0,0 +1,90 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class Command {
+    private static Logger logger = Logger.getLogger(Command.class.getName());
+    private static Pattern ARGUMENT_PATTERN = Pattern.compile("^[a-zA-Z0-9=@%/+:.,_-]+$");
+
+    private final TaskContext context;
+    private final List<String> arguments = new ArrayList<>();
+
+    public Command(TaskContext context) {
+        this.context = context;
+    }
+
+    public Command add(String... arguments) { return add(Arrays.asList(arguments)); }
+    public Command add(List<String> arguments) {
+        this.arguments.addAll(arguments);
+        return this;
+    }
+
+    public ChildProcess spawn(Logger commandLogger) {
+        if (arguments.isEmpty()) {
+            throw new IllegalStateException("No program has been specified");
+        }
+
+        String commandLine = commandLine();
+        if (commandLogger != null) {
+            context.logSystemModification(commandLogger, "Executing command: " + commandLine);
+        }
+
+        // Why isn't this using TaskContext.fileSystem? Because createTempFile assumes
+        // default FileSystem. And Jimfs doesn't support toFile() needed for Redirect below.
+        Path temporaryFile = IOExceptionUtil.uncheck(() -> Files.createTempFile(
+                Command.class.getSimpleName() + "-",
+                ".out"));
+
+        ProcessBuilder builder = new ProcessBuilder(arguments)
+                .redirectError(ProcessBuilder.Redirect.appendTo(temporaryFile.toFile()))
+                .redirectOutput(temporaryFile.toFile());
+        Process process = IOExceptionUtil.uncheck(builder::start);
+
+        return new ChildProcessImpl(process, temporaryFile, commandLine);
+    }
+
+    String commandLine() {
+        return arguments.stream().map(Command::maybeEscapeArgument).collect(Collectors.joining(" "));
+    }
+
+    private static String maybeEscapeArgument(String argument) {
+        if (ARGUMENT_PATTERN.matcher(argument).matches()) {
+            return argument;
+        } else {
+            return escapeArgument(argument);
+        }
+    }
+
+    private static String escapeArgument(String argument) {
+        StringBuilder doubleQuoteEscaped = new StringBuilder(argument.length() + 10);
+
+        for (int i = 0; i < argument.length(); ++i) {
+            char c = argument.charAt(i);
+            switch (c) {
+                case '"':
+                case '\\':
+                    doubleQuoteEscaped.append("\\").append(c);
+                    break;
+                default:
+                    doubleQuoteEscaped.append(c);
+            }
+        }
+
+        return "\"" + doubleQuoteEscaped + "\"";
+    }
+
+    public ChildProcess spawnWithoutLoggingCommand() {
+        return spawn(null);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandException.java
new file mode 100644
index 00000000000..ae84c876b35
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandException.java
@@ -0,0 +1,9 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+@SuppressWarnings("serial")
+public class CommandException extends RuntimeException {
+    public CommandException(String message) {
+        super(message);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi.java
new file mode 100644
index 00000000000..10728334021
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi.java
@@ -0,0 +1,9 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+import java.nio.file.Path;
+import java.util.List;
+
+public interface ProcessApi {
+    ChildProcessImpl spawn(List<String> args, Path outFile);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessImpl.java
new file mode 100644
index 00000000000..d6eb32163ba
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessImpl.java
@@ -0,0 +1,54 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class ProcessImpl implements ProcessApi {
+    private static Pattern ARGUMENT_PATTERN = Pattern.compile("^[a-zA-Z0-9=@%/+:.,_-]+$");
+
+    @Override
+    public ChildProcessImpl spawn(List<String> arguments, Path outFile) {
+        ProcessBuilder builder = new ProcessBuilder(arguments)
+                .redirectError(ProcessBuilder.Redirect.appendTo(outFile.toFile()))
+                .redirectOutput(outFile.toFile());
+        Process process = IOExceptionUtil.uncheck(builder::start);
+        return new ChildProcessImpl(process, outFile, commandLine(arguments));
+    }
+
+    String commandLine(List<String> arguments) {
+        return arguments.stream()
+                .map(ProcessImpl::maybeEscapeArgument)
+                .collect(Collectors.joining(" "));
+    }
+
+    private static String maybeEscapeArgument(String argument) {
+        if (ARGUMENT_PATTERN.matcher(argument).matches()) {
+            return argument;
+        } else {
+            return escapeArgument(argument);
+        }
+    }
+
+    private static String escapeArgument(String argument) {
+        StringBuilder doubleQuoteEscaped = new StringBuilder(argument.length() + 10);
+
+        for (int i = 0; i < argument.length(); ++i) {
+            char c = argument.charAt(i);
+            switch (c) {
+                case '"':
+                case '\\':
+                    doubleQuoteEscaped.append("\\").append(c);
+                    break;
+                default:
+                    doubleQuoteEscaped.append(c);
+            }
+        }
+
+        return "\"" + doubleQuoteEscaped + "\"";
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java
new file mode 100644
index 00000000000..16da9a3b7ca
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
new file mode 100644
index 00000000000..3ca20cba99a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
@@ -0,0 +1,83 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcess;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.Command;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+
+public class Yum {
+    private static Logger logger = Logger.getLogger(Yum.class.getName());
+
+    private final TaskContext taskContext;
+    private final Supplier<Command> commandSupplier;
+    private List<String> packages = new ArrayList<>();
+
+    public Yum(TaskContext taskContext) {
+        this.taskContext = taskContext;
+        this.commandSupplier = () -> new Command(taskContext);
+    }
+
+    /**
+     * @param packages A list of packages, each package being of the form name-1.2.3-1.el7.noarch
+     */
+    public Install install(String... packages) {
+        return new Install(taskContext, Arrays.asList(packages));
+    }
+
+    public class Install {
+        private final TaskContext taskContext;
+        private final List<String> packages;
+        private Optional<String> enabledRepo = Optional.empty();
+
+        public Install(TaskContext taskContext, List<String> packages) {
+            this.taskContext = taskContext;
+            this.packages = packages;
+
+            if (packages.isEmpty()) {
+                throw new IllegalArgumentException("No packages specified");
+            }
+        }
+
+        public Install enableRepo(String repo) {
+            enabledRepo = Optional.of(repo);
+            return this;
+        }
+
+        public boolean converge() {
+            if (packages.stream().allMatch(Yum.this::isInstalled)) {
+                return false;
+            }
+
+            execute();
+            return true;
+        }
+
+        private void execute() {
+            Command command = commandSupplier.get();
+            command.add("yum", "install", "--assumeyes");
+            enabledRepo.ifPresent(repo -> command.add("--enablerepo=" + repo));
+            command.add(packages);
+            command.spawn(logger).waitForTermination().throwIfFailed();
+        }
+    }
+
+    Yum(TaskContext taskContext, Supplier<Command> commandSupplier) {
+        this.taskContext = taskContext;
+        this.commandSupplier = commandSupplier;
+    }
+
+    private boolean isInstalled(String package_) {
+        ChildProcess childProcess = commandSupplier.get()
+                .add("yum", "list", "installed", package_)
+                .spawnWithoutLoggingCommand();
+        childProcess.waitForTermination();
+        return childProcess.exitValue() == 0;
+    }
+}