Merge pull request #4772 from vespa-engine/hakonhall/yum-install

YUM install

10 files changed, 550 insertions, 0 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess.java
new file mode 100644
index 00000000000..9c3333d41ae
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess.java
@@ -0,0 +1,17 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.nio.file.Path;
+
+public interface ChildProcess extends AutoCloseable {
+    ChildProcess waitForTermination();
+    int exitValue();
+    ChildProcess throwIfFailed();
+    String getUtf8Output();
+
+    @Override
+    void close();
+
+    // For testing only
+    Path getProcessOutputPath();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessImpl.java
new file mode 100644
index 00000000000..b9e075dfab7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessImpl.java
@@ -0,0 +1,78 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+
+import java.nio.file.Path;
+
+/**
+ * Represents a forked child process that still exists or has terminated.
+ */
+public class ChildProcessImpl implements ChildProcess {
+    private final Process process;
+    private final Path processOutputPath;
+    private final String commandLine;
+
+    ChildProcessImpl(Process process, Path processOutputPath, String commandLine) {
+        this.process = process;
+        this.processOutputPath = processOutputPath;
+        this.commandLine = commandLine;
+    }
+
+    public String getUtf8Output() {
+        return new UnixPath(processOutputPath).readUtf8File();
+    }
+
+    public ChildProcessImpl waitForTermination() {
+        while (true) {
+            try {
+                process.waitFor();
+            } catch (InterruptedException e) {
+                // ignoring
+                continue;
+            }
+
+            return this;
+        }
+    }
+
+    public int exitValue() {
+        return process.exitValue();
+    }
+
+    public ChildProcess throwIfFailed() {
+        if (process.exitValue() != 0) {
+            throw new CommandException("Execution of program [" + commandLine +
+                    "] failed, stdout/stderr was: <" + suffixOfOutputForLog() + ">");
+        }
+
+        return this;
+    }
+
+    private String suffixOfOutputForLog() {
+        String output = getUtf8Output();
+
+        final int maxTrailingChars = 300;
+        if (output.length() <= maxTrailingChars) {
+            return output;
+        }
+
+        int numSkippedChars = output.length() - maxTrailingChars;
+        output = output.substring(numSkippedChars);
+        return "[" + numSkippedChars + " chars omitted]..." + output;
+    }
+
+    @Override
+    public void close() {
+        if (process.isAlive()) {
+            process.destroyForcibly();
+            waitForTermination();
+        }
+        processOutputPath.toFile().delete();
+    }
+
+    @Override
+    public Path getProcessOutputPath() {
+        return processOutputPath;
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Command.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Command.java
new file mode 100644
index 00000000000..72aa5b0df2d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Command.java
@@ -0,0 +1,90 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class Command {
+    private static Logger logger = Logger.getLogger(Command.class.getName());
+    private static Pattern ARGUMENT_PATTERN = Pattern.compile("^[a-zA-Z0-9=@%/+:.,_-]+$");
+
+    private final TaskContext context;
+    private final List<String> arguments = new ArrayList<>();
+
+    public Command(TaskContext context) {
+        this.context = context;
+    }
+
+    public Command add(String... arguments) { return add(Arrays.asList(arguments)); }
+    public Command add(List<String> arguments) {
+        this.arguments.addAll(arguments);
+        return this;
+    }
+
+    public ChildProcess spawn(Logger commandLogger) {
+        if (arguments.isEmpty()) {
+            throw new IllegalStateException("No program has been specified");
+        }
+
+        String commandLine = commandLine();
+        if (commandLogger != null) {
+            context.logSystemModification(commandLogger, "Executing command: " + commandLine);
+        }
+
+        // Why isn't this using TaskContext.fileSystem? Because createTempFile assumes
+        // default FileSystem. And Jimfs doesn't support toFile() needed for Redirect below.
+        Path temporaryFile = IOExceptionUtil.uncheck(() -> Files.createTempFile(
+                Command.class.getSimpleName() + "-",
+                ".out"));
+
+        ProcessBuilder builder = new ProcessBuilder(arguments)
+                .redirectError(ProcessBuilder.Redirect.appendTo(temporaryFile.toFile()))
+                .redirectOutput(temporaryFile.toFile());
+        Process process = IOExceptionUtil.uncheck(builder::start);
+
+        return new ChildProcessImpl(process, temporaryFile, commandLine);
+    }
+
+    String commandLine() {
+        return arguments.stream().map(Command::maybeEscapeArgument).collect(Collectors.joining(" "));
+    }
+
+    private static String maybeEscapeArgument(String argument) {
+        if (ARGUMENT_PATTERN.matcher(argument).matches()) {
+            return argument;
+        } else {
+            return escapeArgument(argument);
+        }
+    }
+
+    private static String escapeArgument(String argument) {
+        StringBuilder doubleQuoteEscaped = new StringBuilder(argument.length() + 10);
+
+        for (int i = 0; i < argument.length(); ++i) {
+            char c = argument.charAt(i);
+            switch (c) {
+                case '"':
+                case '\\':
+                    doubleQuoteEscaped.append("\\").append(c);
+                    break;
+                default:
+                    doubleQuoteEscaped.append(c);
+            }
+        }
+
+        return "\"" + doubleQuoteEscaped + "\"";
+    }
+
+    public ChildProcess spawnWithoutLoggingCommand() {
+        return spawn(null);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandException.java
new file mode 100644
index 00000000000..ae84c876b35
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandException.java
@@ -0,0 +1,9 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+@SuppressWarnings("serial")
+public class CommandException extends RuntimeException {
+    public CommandException(String message) {
+        super(message);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java
new file mode 100644
index 00000000000..16da9a3b7ca
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
new file mode 100644
index 00000000000..3ca20cba99a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
@@ -0,0 +1,83 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcess;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.Command;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+
+public class Yum {
+    private static Logger logger = Logger.getLogger(Yum.class.getName());
+
+    private final TaskContext taskContext;
+    private final Supplier<Command> commandSupplier;
+    private List<String> packages = new ArrayList<>();
+
+    public Yum(TaskContext taskContext) {
+        this.taskContext = taskContext;
+        this.commandSupplier = () -> new Command(taskContext);
+    }
+
+    /**
+     * @param packages A list of packages, each package being of the form name-1.2.3-1.el7.noarch
+     */
+    public Install install(String... packages) {
+        return new Install(taskContext, Arrays.asList(packages));
+    }
+
+    public class Install {
+        private final TaskContext taskContext;
+        private final List<String> packages;
+        private Optional<String> enabledRepo = Optional.empty();
+
+        public Install(TaskContext taskContext, List<String> packages) {
+            this.taskContext = taskContext;
+            this.packages = packages;
+
+            if (packages.isEmpty()) {
+                throw new IllegalArgumentException("No packages specified");
+            }
+        }
+
+        public Install enableRepo(String repo) {
+            enabledRepo = Optional.of(repo);
+            return this;
+        }
+
+        public boolean converge() {
+            if (packages.stream().allMatch(Yum.this::isInstalled)) {
+                return false;
+            }
+
+            execute();
+            return true;
+        }
+
+        private void execute() {
+            Command command = commandSupplier.get();
+            command.add("yum", "install", "--assumeyes");
+            enabledRepo.ifPresent(repo -> command.add("--enablerepo=" + repo));
+            command.add(packages);
+            command.spawn(logger).waitForTermination().throwIfFailed();
+        }
+    }
+
+    Yum(TaskContext taskContext, Supplier<Command> commandSupplier) {
+        this.taskContext = taskContext;
+        this.commandSupplier = commandSupplier;
+    }
+
+    private boolean isInstalled(String package_) {
+        ChildProcess childProcess = commandSupplier.get()
+                .add("yum", "list", "installed", package_)
+                .spawnWithoutLoggingCommand();
+        childProcess.waitForTermination();
+        return childProcess.exitValue() == 0;
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandTest.java
new file mode 100644
index 00000000000..373c75eba59
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandTest.java
@@ -0,0 +1,87 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import org.junit.Test;
+
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.util.logging.Logger;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class CommandTest {
+    @Test
+    public void testCommand() {
+        TaskContext taskContext = mock(TaskContext.class);
+        Logger logger = mock(Logger.class);
+
+        Command command = new Command(taskContext).add("bash", "-c", "ls /bin/bash");
+        Path outputFile;
+        // Assumes /bin/bash exists on all hosts running this test.
+        try (ChildProcess childProcess = command.spawn(logger)) {
+            verify(taskContext).logSystemModification(eq(logger), any());
+
+            outputFile = childProcess.getProcessOutputPath();
+            int exitValue = childProcess.waitForTermination().exitValue();
+            assertEquals(0, exitValue);
+            childProcess.throwIfFailed();
+            String output = childProcess.getUtf8Output().trim();
+            assertEquals("/bin/bash", output);
+            assertTrue(outputFile.toFile().exists());
+        }
+
+        assertFalse(outputFile.toFile().exists());
+    }
+
+    @Test(expected = UncheckedIOException.class)
+    public void noSuchProgram() {
+        TaskContext taskContext = mock(TaskContext.class);
+        Logger logger = mock(Logger.class);
+
+        Command command = new Command(taskContext).add("thisprogRamDoes-not-exist");
+        try (ChildProcess childProcess = command.spawn(logger)) {
+            dummyToRemoveWarning(childProcess);
+        }
+
+        fail();
+    }
+
+    private void dummyToRemoveWarning(ChildProcess childProcess) { }
+
+    @Test
+    public void argumentEscape() {
+        TaskContext taskContext = mock(TaskContext.class);
+        Command command = new Command(taskContext).add("b", "\" \\ foo", "bar x", "");
+        assertEquals("b \"\\\" \\\\ foo\" \"bar x\" \"\"", command.commandLine());
+    }
+
+    @Test
+    public void failingProgram() {
+        TaskContext taskContext = mock(TaskContext.class);
+        Logger logger = mock(Logger.class);
+
+        Command command = new Command(taskContext)
+                .add("bash", "-c", "echo foo; echo bar >&2; exit 1");
+        Path outputFile;
+        try (ChildProcess childProcess = command.spawn(logger)) {
+            try {
+                childProcess.waitForTermination().throwIfFailed();
+                fail();
+            } catch (CommandException e) {
+                assertEquals("Execution of program [bash -c \"echo foo; echo bar >&2; exit 1\"] failed, stdout/stderr was: <foo\n" +
+                                "bar\n" +
+                                ">",
+                        e.getMessage());
+            }
+        }
+
+    }
+}
\ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommand.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommand.java
new file mode 100644
index 00000000000..59c853f949d
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommand.java
@@ -0,0 +1,71 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.file.Path;
+import java.util.logging.Logger;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class TestCommand extends Command {
+    private final String expectedCommandLine;
+    private final ChildProcess childProcess;
+
+    private boolean invoked = false;
+
+    public TestCommand(TaskContext context,
+                       String expectedCommandLine,
+                       int exitValue,
+                       String out) {
+        super(context);
+        this.expectedCommandLine = expectedCommandLine;
+        this.childProcess = new ChildProcess() {
+            @Override
+            public ChildProcess waitForTermination() {
+                return this;
+            }
+
+            @Override
+            public int exitValue() {
+                return exitValue;
+            }
+
+            @Override
+            public ChildProcess throwIfFailed() {
+                if (exitValue != 0) {
+                    throw new CommandException("exited with " + exitValue);
+                }
+                return this;
+            }
+
+            @Override
+            public String getUtf8Output() {
+                return out;
+            }
+
+            @Override
+            public void close() { }
+
+            @Override
+            public Path getProcessOutputPath() { return null; }
+        };
+    }
+
+    @Override
+    public ChildProcess spawn(Logger commandLogger) {
+        assertFalse(invoked);
+        invoked = true;
+
+        assertEquals(expectedCommandLine, commandLine());
+
+        return childProcess;
+    }
+
+    public void verifyInvocation() {
+        if (!invoked) {
+            throw new IllegalStateException("Command not invoked: " + expectedCommandLine);
+        }
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommandSupplier.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommandSupplier.java
new file mode 100644
index 00000000000..1c900604260
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommandSupplier.java
@@ -0,0 +1,43 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class TestCommandSupplier implements Supplier<Command> {
+    private final TaskContext taskContext;
+    private final List<TestCommand> expectedInvocations = new ArrayList<>();
+    private int index = 0;
+
+    public TestCommandSupplier(TaskContext taskContext) {
+        this.taskContext = taskContext;
+    }
+
+    public TestCommandSupplier expectCommand(String commandLine, int exitValue, String out) {
+        expectedInvocations.add(new TestCommand(taskContext, commandLine, exitValue, out));
+        return this;
+    }
+
+    @Override
+    public Command get() {
+        if (index >= expectedInvocations.size()) {
+            throw new IllegalStateException("Too many command invocations");
+        }
+
+        return expectedInvocations.get(index++);
+    }
+
+    public void verifyInvocations() {
+        if (index != expectedInvocations.size()) {
+            throw new IllegalStateException("Received only " + index +
+                    " command invocations: expected " + expectedInvocations.size());
+        }
+
+        for (int i = 0; i < index; ++i) {
+            expectedInvocations.get(i).verifyInvocation();
+        }
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
new file mode 100644
index 00000000000..d852be26229
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
@@ -0,0 +1,67 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandException;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestCommandSupplier;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+public class YumTest {
+    @Test
+    public void testAlreadyInstalled() {
+        TaskContext taskContext = mock(TaskContext.class);
+        TestCommandSupplier commandSupplier = new TestCommandSupplier(taskContext);
+
+        commandSupplier.expectCommand("yum list installed package-1", 0, "");
+        commandSupplier.expectCommand("yum list installed package-2", 0, "");
+
+        Yum yum = new Yum(taskContext, commandSupplier);
+        yum.install("package-1", "package-2")
+                .enableRepo("repo-name")
+                .converge();
+
+        commandSupplier.verifyInvocations();
+    }
+
+    @Test
+    public void testInstall() {
+        TaskContext taskContext = mock(TaskContext.class);
+        TestCommandSupplier commandSupplier = new TestCommandSupplier(taskContext);
+
+        commandSupplier.expectCommand("yum list installed package-1", 0, "");
+        commandSupplier.expectCommand("yum list installed package-2", 1, "");
+        commandSupplier.expectCommand(
+                "yum install --assumeyes --enablerepo=repo-name package-1 package-2",
+                0,
+                "");
+
+        Yum yum = new Yum(taskContext, commandSupplier);
+        yum.install("package-1", "package-2")
+                .enableRepo("repo-name")
+                .converge();
+
+        commandSupplier.verifyInvocations();
+    }
+
+    @Test(expected = CommandException.class)
+    public void testFailedInstall() {
+        TaskContext taskContext = mock(TaskContext.class);
+        TestCommandSupplier commandSupplier = new TestCommandSupplier(taskContext);
+
+        commandSupplier.expectCommand("yum list installed package-1", 0, "");
+        commandSupplier.expectCommand("yum list installed package-2", 1, "");
+        commandSupplier.expectCommand(
+                "yum install --assumeyes --enablerepo=repo-name package-1 package-2",
+                1,
+                "error");
+
+        Yum yum = new Yum(taskContext, commandSupplier);
+        yum.install("package-1", "package-2")
+                .enableRepo("repo-name")
+                .converge();
+        fail();
+    }
+}
\ No newline at end of file