diff options
Diffstat (limited to 'node-admin')
38 files changed, 1381 insertions, 94 deletions
diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 7b3b787b503..161769a4edf 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -116,12 +116,11 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.google.jimfs</groupId> - <artifactId>jimfs</artifactId> - <scope>test</scope> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity</artifactId> + <scope>compile</scope> </dependency> </dependencies> - <build> <plugins> <plugin> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java index 9def627e87f..87491367514 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java @@ -18,4 +18,6 @@ public interface TaskContext { FileSystem fileSystem(); void logSystemModification(Logger logger, String actionDescription); + + default boolean executeSubtask(IdempotentTask task) { return false; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java new file mode 100644 index 00000000000..dfcaba7c4bb --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java @@ -0,0 +1,53 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.provider; + +import javax.annotation.concurrent.ThreadSafe; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Class to make it easier to implement a NodeAdminDebugHandler: + * - Forward to sub-NodeAdminDebugHandlers with addHandler, + * - Specify constants with addConstant + * - Forwarding to methods that dynamically build debug objects with addThreadSafeSupplier. + * + * @author hakonhall + */ +@ThreadSafe +public class DebugHandlerHelper implements NodeAdminDebugHandler { + private Object monitor = new Object(); + private final ConcurrentMap<String, Supplier<Object>> suppliers = new ConcurrentHashMap<>(); + + public void addThreadSafeSupplier(String name, Supplier<Object> threadSafeSupplier) { + Supplier<Object> previousSupplier = suppliers.putIfAbsent(name, threadSafeSupplier); + if (previousSupplier != null) { + throw new IllegalArgumentException(name + " is already registered"); + } + } + + public void addHandler(String name, NodeAdminDebugHandler handler) { + addThreadSafeSupplier(name, () -> handler.getDebugPage()); + } + + public void addConstant(String name, String value) { + addThreadSafeSupplier(name, () -> value); + } + + public void remove(String name) { + Supplier<Object> supplier = suppliers.remove(name); + if (supplier == null) { + throw new IllegalArgumentException(name + " is not registered"); + } + } + + @Override + public Map<String, Object> getDebugPage() { + return suppliers.entrySet().stream().collect(Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue().get())); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java new file mode 100644 index 00000000000..7b5eaa2f326 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java @@ -0,0 +1,20 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.provider; + +import javax.annotation.concurrent.ThreadSafe; +import java.util.Map; + +/** + * Interface for supporting debug info to introspect e.g. internal state. + * + * @author hakonhall + */ +@ThreadSafe +public interface NodeAdminDebugHandler { + /** + * The Object in the map values must be serializable with Jackson's ObjectMapper. + * May be called concurrently by different threads. + */ + Map<String, Object> getDebugPage(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java index 755e1301c12..841f464e014 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java @@ -1,9 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.provider; -import java.util.Map; +import javax.annotation.concurrent.ThreadSafe; -public interface NodeAdminStateUpdater { +@ThreadSafe +public interface NodeAdminStateUpdater extends NodeAdminDebugHandler { enum State { TRANSITIONING, RESUMED, SUSPENDED_NODE_ADMIN, SUSPENDED} /** @@ -12,6 +13,4 @@ public interface NodeAdminStateUpdater { * has converged. */ boolean setResumeStateAndCheckIfResumed(State wantedState); - - Map<String, Object> getDebugPage(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java new file mode 100644 index 00000000000..3910398a040 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java @@ -0,0 +1,27 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Instant; + +/** + * This wrapper around PosixFileAttributes. + * + * @author hakonhall + */ +public class FileAttributes { + private final PosixFileAttributes attributes; + + FileAttributes(PosixFileAttributes attributes) { + this.attributes = attributes; + } + + public Instant lastModifiedTime() { return attributes.lastModifiedTime().toInstant(); } + public String owner() { return attributes.owner().getName(); } + public String group() { return attributes.group().getName(); } + public String permissions() { return PosixFilePermissions.toString(attributes.permissions()); } + public boolean isRegularFile() { return attributes.isRegularFile(); } + public boolean isDirectory() { return attributes.isDirectory(); } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java new file mode 100644 index 00000000000..12a9609f89c --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java @@ -0,0 +1,44 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.util.Optional; + +// @ThreadUnsafe +public class FileAttributesCache { + private final UnixPath path; + + private Optional<FileAttributes> attributes = Optional.empty(); + + public FileAttributesCache(UnixPath path) { + this.path = path; + } + + public FileAttributes get() { + if (!attributes.isPresent()) { + attributes = Optional.of(path.getAttributes()); + } + + return attributes.get(); + } + + public FileAttributes forceGet() { + attributes = Optional.empty(); + return get(); + } + + public boolean exists() { + if (attributes.isPresent()) { + return true; + } + + Optional<FileAttributes> attributes = path.getAttributesIfExists(); + if (attributes.isPresent()) { + // Might as well update this.attributes + this.attributes = attributes; + return true; + } else { + return false; + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java new file mode 100644 index 00000000000..ca79e8bb113 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.time.Instant; +import java.util.Optional; + +/** + * Class to avoid repeated reads of file content when the file seldom changes. + * + * @author hakonhall + */ +class FileContentCache { + private final UnixPath path; + + private Optional<String> value = Optional.empty(); + private Optional<Instant> modifiedTime = Optional.empty(); + + FileContentCache(UnixPath path) { + this.path = path; + } + + String get(Instant lastModifiedTime) { + if (!value.isPresent() || lastModifiedTime.compareTo(modifiedTime.get()) > 0) { + value = Optional.of(path.readUtf8File()); + modifiedTime = Optional.of(lastModifiedTime); + } + + return value.get(); + } + + void updateWith(String content, Instant modifiedTime) { + this.value = Optional.of(content); + this.modifiedTime = Optional.of(modifiedTime); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java new file mode 100644 index 00000000000..d8b8aadfff7 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java @@ -0,0 +1,118 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.logging.Logger; + +/** + * Class to minimize resource usage with repetitive and mostly identical, idempotent, and + * mutating file operations, e.g. setting file content, setting owner, etc. + * + * Only changes to the file is logged. + * + * @author hakohall + */ +// @ThreadUnsafe +public class FileSync { + private static final Logger logger = Logger.getLogger(FileSync.class.getName()); + + private final UnixPath path; + private final FileContentCache contentCache; + + public FileSync(Path path) { + this.path = new UnixPath(path); + this.contentCache = new FileContentCache(this.path); + } + + /** + * CPU, I/O, and memory usage is optimized for repeated calls with the same arguments. + * @return true if the system was modified: content was written, or owner was set, etc. + * system is only modified if necessary (different). + */ + public boolean convergeTo(TaskContext taskContext, PartialFileData partialFileData) { + FileAttributesCache currentAttributes = new FileAttributesCache(path); + + boolean modifiedSystem = false; + + modifiedSystem |= maybeUpdateContent(taskContext, partialFileData.getContent(), currentAttributes); + + modifiedSystem |= convergeAttribute( + taskContext, + "owner", + partialFileData.getOwner(), + () -> currentAttributes.get().owner(), + path::setOwner); + + modifiedSystem |= convergeAttribute( + taskContext, + "group", + partialFileData.getGroup(), + () -> currentAttributes.get().group(), + path::setGroup); + + modifiedSystem |= convergeAttribute( + taskContext, + "permissions", + partialFileData.getPermissions(), + () -> currentAttributes.get().permissions(), + path::setPermissions); + + return modifiedSystem; + } + + private boolean convergeAttribute(TaskContext taskContext, + String attributeName, + Optional<String> wantedValue, + Supplier<String> currentValueSupplier, + Consumer<String> valueSetter) { + if (!wantedValue.isPresent()) { + return false; + } + + String currentValue = currentValueSupplier.get(); + if (Objects.equals(wantedValue.get(), currentValue)) { + return false; + } else { + String actionDescription = String.format("Changing %s of %s from %s to %s", + attributeName, + path, + currentValue, + wantedValue.get()); + taskContext.logSystemModification(logger, actionDescription); + valueSetter.accept(wantedValue.get()); + return true; + } + } + + private boolean maybeUpdateContent(TaskContext taskContext, + Optional<String> content, + FileAttributesCache currentAttributes) { + if (!content.isPresent()) { + return false; + } + + if (!currentAttributes.exists()) { + taskContext.logSystemModification(logger, "Creating file " + path); + path.createParents(); + path.writeUtf8File(content.get()); + contentCache.updateWith(content.get(), currentAttributes.forceGet().lastModifiedTime()); + return true; + } + + if (Objects.equals(content.get(), contentCache.get(currentAttributes.get().lastModifiedTime()))) { + return false; + } else { + taskContext.logSystemModification(logger, "Patching file " + path); + path.writeUtf8File(content.get()); + contentCache.updateWith(content.get(), currentAttributes.forceGet().lastModifiedTime()); + return true; + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java index 60a7b3482b2..58518ae5a15 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java @@ -6,56 +6,53 @@ import org.glassfish.jersey.internal.util.Producer; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Optional; -import java.util.logging.Logger; +/** + * Write a file + * + * @author hakonhall + */ public class FileWriter { - private static final Logger logger = Logger.getLogger(FileWriter.class.getName()); - private final Path path; + private final FileSync fileSync; + private final PartialFileData.Builder fileDataBuilder = PartialFileData.builder(); private final Producer<String> contentProducer; - private Optional<String> owner = Optional.empty(); - private Optional<String> group = Optional.empty(); - private Optional<String> permissions = Optional.empty(); + private boolean overwriteExistingFile = true; public FileWriter(Path path, Producer<String> contentProducer) { this.path = path; + this.fileSync = new FileSync(path); this.contentProducer = contentProducer; } public FileWriter withOwner(String owner) { - this.owner = Optional.of(owner); + fileDataBuilder.withOwner(owner); return this; } public FileWriter withGroup(String group) { - this.group = Optional.of(group); + fileDataBuilder.withGroup(group); return this; } public FileWriter withPermissions(String permissions) { - this.permissions = Optional.of(permissions); + fileDataBuilder.withPermissions(permissions); + return this; + } + + public FileWriter onlyIfFileDoesNotAlreadyExist() { + overwriteExistingFile = false; return this; } public boolean converge(TaskContext context) { - // TODO: Only return false if content, permission, etc would be unchanged. - if (Files.isRegularFile(path)) { + if (!overwriteExistingFile && Files.isRegularFile(path)) { return false; } - context.logSystemModification(logger,"Writing file " + path); - - String content = contentProducer.call(); - - UnixPath unixPath = new UnixPath(path); - unixPath.createParents(); - unixPath.writeUtf8File(content); - permissions.ifPresent(unixPath::setPermissions); - owner.ifPresent(unixPath::setOwner); - group.ifPresent(unixPath::setGroup); - - return true; + fileDataBuilder.withContent(contentProducer.call()); + PartialFileData fileData = fileDataBuilder.create(); + return fileSync.convergeTo(context, fileData); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java index dee5525d42a..9bcf601c262 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java @@ -3,7 +3,12 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.NoSuchFileException; +import java.util.Optional; +/** + * @author hakonhall + */ public class IOExceptionUtil { public static <T> void uncheck(RunnableThrowingIOException<T> runnable) { try { @@ -31,4 +36,20 @@ public class IOExceptionUtil { public interface RunnableThrowingIOException<T> { void run() throws IOException; } + + /** + * Useful if it's not known whether a file or directory exists, in case e.g. + * NoSuchFileException is thrown and the caller wants an Optional.empty() in that case. + */ + public static <T> Optional<T> ifExists(SupplierThrowingIOException<T> supplier) { + try { + return Optional.ofNullable(uncheck(supplier)); + } catch (UncheckedIOException e) { + if (e.getCause() instanceof NoSuchFileException) { + return Optional.empty(); + } + + throw e; + } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java new file mode 100644 index 00000000000..b931a374230 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java @@ -0,0 +1,64 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.util.Optional; + +/** + * Represents a subset of a file's content, owner, group, and permissions. + * + * @author hakonhall + */ +// @Immutable +public class PartialFileData { + private final Optional<String> content; + private final Optional<String> owner; + private final Optional<String> group; + private final Optional<String> permissions; + + public static Builder builder() { + return new Builder(); + } + + public PartialFileData(Optional<String> content, + Optional<String> owner, + Optional<String> group, + Optional<String> permissions) { + this.content = content; + this.owner = owner; + this.group = group; + this.permissions = permissions; + } + + public Optional<String> getContent() { + return content; + } + + public Optional<String> getOwner() { + return owner; + } + + public Optional<String> getGroup() { + return group; + } + + public Optional<String> getPermissions() { + return permissions; + } + + public static class Builder { + private Optional<String> content = Optional.empty(); + private Optional<String> owner = Optional.empty(); + private Optional<String> group = Optional.empty(); + private Optional<String> permissions = Optional.empty(); + + public Builder withContent(String content) { this.content = Optional.of(content); return this; } + public Builder withOwner(String owner) { this.owner = Optional.of(owner); return this; } + public Builder withGroup(String group) { this.group = Optional.of(group); return this; } + public Builder withPermissions(String permissions) { this.permissions = Optional.of(permissions); return this; } + + public PartialFileData create() { + return new PartialFileData(content, owner, group, permissions); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFile.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFile.java new file mode 100644 index 00000000000..e4dd5cf5d9c --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFile.java @@ -0,0 +1,47 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.apache.velocity.Template; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; +import org.apache.velocity.app.VelocityEngine; + +import java.io.StringWriter; +import java.nio.file.Path; + +/** + * Make a file based on a Velocity template file. + * + * @author hakonhall + */ +public class TemplateFile { + private final Path templatePath; + private final VelocityEngine velocityEngine; + private final VelocityContext velocityContext = new VelocityContext(); + + public TemplateFile(Path templatePath) { + this.templatePath = templatePath; + velocityEngine = new VelocityEngine(); + velocityEngine.addProperty( + Velocity.RUNTIME_LOG_LOGSYSTEM_CLASS, + "org.apache.velocity.runtime.log.NullLogSystem"); + velocityEngine.addProperty(Velocity.FILE_RESOURCE_LOADER_PATH, templatePath.getParent().toString()); + velocityEngine.init(); + } + + public TemplateFile set(String name, String value) { + velocityContext.put(name, value); + return this; + } + + public FileWriter getFileWriterTo(Path destinationPath) { + return new FileWriter(destinationPath, this::render); + } + + private String render() { + Template template = velocityEngine.getTemplate(templatePath.getFileName().toString(), "UTF-8"); + StringWriter writer = new StringWriter(); + template.merge(velocityContext, writer); + return writer.toString(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 606f8cfb06e..aaffea05d1e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -1,8 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.file; -import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.OpenOption; @@ -16,8 +14,16 @@ import java.nio.file.attribute.PosixFilePermissions; import java.nio.file.attribute.UserPrincipal; import java.nio.file.attribute.UserPrincipalLookupService; import java.time.Instant; +import java.util.Optional; import java.util.Set; +import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck; + +/** + * Thin wrapper around java.nio.file.Path, especially nice for UNIX-specific features. + * + * @author hakonhall + */ // @Immutable public class UnixPath { private final Path path; @@ -34,8 +40,14 @@ public class UnixPath { return path; } - public void createParents() { - uncheck(() -> Files.createDirectories(path.getParent())); + public boolean createParents() { + Path parent = path.getParent(); + if (Files.isDirectory(parent)) { + return false; + } + + uncheck(() -> Files.createDirectories(parent)); + return true; } public String readUtf8File() { @@ -49,7 +61,7 @@ public class UnixPath { } public String getPermissions() { - return PosixFilePermissions.toString(getAttributes().permissions()); + return getAttributes().permissions(); } /** @@ -69,7 +81,7 @@ public class UnixPath { } public String getOwner() { - return getAttributes().owner().getName(); + return getAttributes().owner(); } public void setOwner(String owner) { @@ -79,7 +91,7 @@ public class UnixPath { } public String getGroup() { - return getAttributes().group().getName(); + return getAttributes().group(); } public void setGroup(String group) { @@ -89,37 +101,21 @@ public class UnixPath { } public Instant getLastModifiedTime() { - return uncheck(() -> Files.getLastModifiedTime(path)).toInstant(); + return getAttributes().lastModifiedTime(); } - private PosixFileAttributes getAttributes() { - return uncheck(() -> + public FileAttributes getAttributes() { + PosixFileAttributes attributes = uncheck(() -> Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes()); + return new FileAttributes(attributes); } - @FunctionalInterface - private interface SupplierThrowingIOException<T> { - T get() throws IOException; + public Optional<FileAttributes> getAttributesIfExists() { + return IOExceptionUtil.ifExists(() -> getAttributes()); } - private static <T> T uncheck(SupplierThrowingIOException<T> supplier) { - try { - return supplier.get(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - @FunctionalInterface - private interface RunnableThrowingIOException<T> { - void run() throws IOException; - } - - private static <T> void uncheck(RunnableThrowingIOException<T> runnable) { - try { - runnable.run(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + @Override + public String toString() { + return path.toString(); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess.java new file mode 100644 index 00000000000..00bcca71970 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess.java @@ -0,0 +1,20 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import java.nio.file.Path; + +/** + * @author hakonhall + */ +public interface ChildProcess extends AutoCloseable { + ChildProcess waitForTermination(); + int exitValue(); + ChildProcess throwIfFailed(); + String getUtf8Output(); + + @Override + void close(); + + // For testing only + Path getProcessOutputPath(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessImpl.java new file mode 100644 index 00000000000..367688f0bb4 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessImpl.java @@ -0,0 +1,80 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; + +import java.nio.file.Path; + +/** + * Represents a forked child process that still exists or has terminated. + * + * @author hakonhall + */ +public class ChildProcessImpl implements ChildProcess { + private final Process process; + private final Path processOutputPath; + private final String commandLine; + + ChildProcessImpl(Process process, Path processOutputPath, String commandLine) { + this.process = process; + this.processOutputPath = processOutputPath; + this.commandLine = commandLine; + } + + public String getUtf8Output() { + return new UnixPath(processOutputPath).readUtf8File(); + } + + public ChildProcessImpl waitForTermination() { + while (true) { + try { + process.waitFor(); + } catch (InterruptedException e) { + // ignoring + continue; + } + + return this; + } + } + + public int exitValue() { + return process.exitValue(); + } + + public ChildProcess throwIfFailed() { + if (process.exitValue() != 0) { + throw new CommandException("Execution of program [" + commandLine + + "] failed, stdout/stderr was: <" + suffixOfOutputForLog() + ">"); + } + + return this; + } + + private String suffixOfOutputForLog() { + String output = getUtf8Output(); + + final int maxTrailingChars = 300; + if (output.length() <= maxTrailingChars) { + return output; + } + + int numSkippedChars = output.length() - maxTrailingChars; + output = output.substring(numSkippedChars); + return "[" + numSkippedChars + " chars omitted]..." + output; + } + + @Override + public void close() { + if (process.isAlive()) { + process.destroyForcibly(); + waitForTermination(); + } + processOutputPath.toFile().delete(); + } + + @Override + public Path getProcessOutputPath() { + return processOutputPath; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Command.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Command.java new file mode 100644 index 00000000000..049490f2705 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Command.java @@ -0,0 +1,95 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Class to fork and exec a program, and gets its exit status and output. + * + * @author hakonhall + */ +public class Command { + private static Logger logger = Logger.getLogger(Command.class.getName()); + private static Pattern ARGUMENT_PATTERN = Pattern.compile("^[a-zA-Z0-9=@%/+:.,_-]+$"); + + private final TaskContext context; + private final List<String> arguments = new ArrayList<>(); + + public Command(TaskContext context) { + this.context = context; + } + + public Command add(String... arguments) { return add(Arrays.asList(arguments)); } + public Command add(List<String> arguments) { + this.arguments.addAll(arguments); + return this; + } + + public ChildProcess spawn(Logger commandLogger) { + if (arguments.isEmpty()) { + throw new IllegalStateException("No program has been specified"); + } + + String commandLine = commandLine(); + if (commandLogger != null) { + context.logSystemModification(commandLogger, "Executing command: " + commandLine); + } + + // Why isn't this using TaskContext.fileSystem? Because createTempFile assumes + // default FileSystem. And Jimfs doesn't support toFile() needed for Redirect below. + Path temporaryFile = IOExceptionUtil.uncheck(() -> Files.createTempFile( + Command.class.getSimpleName() + "-", + ".out")); + + ProcessBuilder builder = new ProcessBuilder(arguments) + .redirectError(ProcessBuilder.Redirect.appendTo(temporaryFile.toFile())) + .redirectOutput(temporaryFile.toFile()); + Process process = IOExceptionUtil.uncheck(builder::start); + + return new ChildProcessImpl(process, temporaryFile, commandLine); + } + + String commandLine() { + return arguments.stream().map(Command::maybeEscapeArgument).collect(Collectors.joining(" ")); + } + + private static String maybeEscapeArgument(String argument) { + if (ARGUMENT_PATTERN.matcher(argument).matches()) { + return argument; + } else { + return escapeArgument(argument); + } + } + + private static String escapeArgument(String argument) { + StringBuilder doubleQuoteEscaped = new StringBuilder(argument.length() + 10); + + for (int i = 0; i < argument.length(); ++i) { + char c = argument.charAt(i); + switch (c) { + case '"': + case '\\': + doubleQuoteEscaped.append("\\").append(c); + break; + default: + doubleQuoteEscaped.append(c); + } + } + + return "\"" + doubleQuoteEscaped + "\""; + } + + public ChildProcess spawnWithoutLoggingCommand() { + return spawn(null); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandException.java new file mode 100644 index 00000000000..148f2102ddf --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandException.java @@ -0,0 +1,12 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +/** + * @author hakonhall + */ +@SuppressWarnings("serial") +public class CommandException extends RuntimeException { + public CommandException(String message) { + super(message); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java new file mode 100644 index 00000000000..16da9a3b7ca --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java index 9ca1c0286f9..2b1cbbed974 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java @@ -7,8 +7,11 @@ import com.yahoo.vespa.hosted.node.admin.task.util.file.FileWriter; import java.nio.file.Path; import java.util.regex.Pattern; +/** + * @author hakonhall + */ public class AddYumRepo { - private static final Pattern REPOSITORY_ID_PATTERN = Pattern.compile("^[a-zA-Z_-]+$"); + private static final Pattern REPOSITORY_ID_PATTERN = Pattern.compile("^[a-zA-Z0-9_-]+$"); private final String repositoryId; // e.g. "platform_rpms-latest" private final String name; // e.g. "Platform RPM Latest Repo" @@ -32,7 +35,8 @@ public class AddYumRepo { FileWriter fileWriter = new FileWriter(path, this::getRepoFileContent) .withOwner("root") .withGroup("root") - .withPermissions("rw-r--r--"); + .withPermissions("rw-r--r--") + .onlyIfFileDoesNotAlreadyExist(); return fileWriter.converge(context); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java new file mode 100644 index 00000000000..c1514f1056b --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java @@ -0,0 +1,86 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.yum; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcess; +import com.yahoo.vespa.hosted.node.admin.task.util.process.Command; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.logging.Logger; + +/** + * @author hakonhall + */ +public class Yum { + private static Logger logger = Logger.getLogger(Yum.class.getName()); + + private final TaskContext taskContext; + private final Supplier<Command> commandSupplier; + private List<String> packages = new ArrayList<>(); + + public Yum(TaskContext taskContext) { + this.taskContext = taskContext; + this.commandSupplier = () -> new Command(taskContext); + } + + /** + * @param packages A list of packages, each package being of the form name-1.2.3-1.el7.noarch + */ + public Install install(String... packages) { + return new Install(taskContext, Arrays.asList(packages)); + } + + public class Install { + private final TaskContext taskContext; + private final List<String> packages; + private Optional<String> enabledRepo = Optional.empty(); + + public Install(TaskContext taskContext, List<String> packages) { + this.taskContext = taskContext; + this.packages = packages; + + if (packages.isEmpty()) { + throw new IllegalArgumentException("No packages specified"); + } + } + + public Install enableRepo(String repo) { + enabledRepo = Optional.of(repo); + return this; + } + + public boolean converge() { + if (packages.stream().allMatch(Yum.this::isInstalled)) { + return false; + } + + execute(); + return true; + } + + private void execute() { + Command command = commandSupplier.get(); + command.add("yum", "install", "--assumeyes"); + enabledRepo.ifPresent(repo -> command.add("--enablerepo=" + repo)); + command.add(packages); + command.spawn(logger).waitForTermination().throwIfFailed(); + } + } + + Yum(TaskContext taskContext, Supplier<Command> commandSupplier) { + this.taskContext = taskContext; + this.commandSupplier = commandSupplier; + } + + private boolean isInstalled(String package_) { + ChildProcess childProcess = commandSupplier.get() + .add("yum", "list", "installed", package_) + .spawnWithoutLoggingCommand(); + childProcess.waitForTermination(); + return childProcess.exitValue() == 0; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java index fbcaf701c6f..84db5840909 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java @@ -1,3 +1,4 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.util; import java.io.FileInputStream; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SelfCloseableHttpClient.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SelfCloseableHttpClient.java index ddb473d348c..8e516729aff 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SelfCloseableHttpClient.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SelfCloseableHttpClient.java @@ -1,3 +1,4 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.util; import com.yahoo.log.LogLevel; diff --git a/node-admin/src/main/resources/configdefinitions/config-server.def b/node-admin/src/main/resources/configdefinitions/config-server.def index e4265e618a1..0de79160277 100644 --- a/node-admin/src/main/resources/configdefinitions/config-server.def +++ b/node-admin/src/main/resources/configdefinitions/config-server.def @@ -1,4 +1,4 @@ -# Copyright 2018 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=vespa.hosted.node.admin.config hosts[] string diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java new file mode 100644 index 00000000000..723b9f0df8a --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java @@ -0,0 +1,35 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.provider; + +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class DebugHandlerHelperTest { + @Test + public void trivial() { + DebugHandlerHelper helper = new DebugHandlerHelper(); + helper.addConstant("constant-key", "constant-value"); + + NodeAdminDebugHandler handler = new NodeAdminDebugHandler() { + @Override + public Map<String, Object> getDebugPage() { + return Collections.singletonMap("handler-value-key", "handler-value-value"); + } + }; + helper.addHandler("handler-key", handler); + + helper.addThreadSafeSupplier("supplier-key", () -> "supplier-value"); + + assertEquals("{" + + "supplier-key=supplier-value, " + + "handler-key={handler-value-key=handler-value-value}, " + + "constant-key=constant-value" + + "}", + helper.getDebugPage().toString()); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java new file mode 100644 index 00000000000..9224faf1c6f --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java @@ -0,0 +1,38 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class FileAttributesCacheTest { + @Test + public void exists() throws Exception { + UnixPath unixPath = mock(UnixPath.class); + FileAttributesCache cache = new FileAttributesCache(unixPath); + + when(unixPath.getAttributesIfExists()).thenReturn(Optional.empty()); + assertFalse(cache.exists()); + verify(unixPath, times(1)).getAttributesIfExists(); + verifyNoMoreInteractions(unixPath); + + FileAttributes attributes = mock(FileAttributes.class); + when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes)); + assertTrue(cache.exists()); + verify(unixPath, times(1 + 1)).getAttributesIfExists(); + verifyNoMoreInteractions(unixPath); + + assertEquals(attributes, cache.get()); + verifyNoMoreInteractions(unixPath); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java new file mode 100644 index 00000000000..677dd048445 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java @@ -0,0 +1,58 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.Test; + +import java.time.Instant; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class FileContentCacheTest { + private final UnixPath unixPath = mock(UnixPath.class); + private final FileContentCache cache = new FileContentCache(unixPath); + + @Test + public void get() throws Exception { + when(unixPath.readUtf8File()).thenReturn("content"); + assertEquals("content", cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1)).readUtf8File(); + verifyNoMoreInteractions(unixPath); + + // cache hit + assertEquals("content", cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1)).readUtf8File(); + verifyNoMoreInteractions(unixPath); + + // cache miss + when(unixPath.readUtf8File()).thenReturn("new-content"); + assertEquals("new-content", cache.get(Instant.ofEpochMilli(1))); + verify(unixPath, times(1 + 1)).readUtf8File(); + verifyNoMoreInteractions(unixPath); + + // cache hit both at times 0 and 1 + assertEquals("new-content", cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1 + 1)).readUtf8File(); + verifyNoMoreInteractions(unixPath); + assertEquals("new-content", cache.get(Instant.ofEpochMilli(1))); + verify(unixPath, times(1 + 1)).readUtf8File(); + verifyNoMoreInteractions(unixPath); + } + + @Test + public void updateWith() throws Exception { + cache.updateWith("content", Instant.ofEpochMilli(2)); + assertEquals("content", cache.get(Instant.ofEpochMilli(2))); + verifyNoMoreInteractions(unixPath); + + cache.updateWith("new-content", Instant.ofEpochMilli(4)); + assertEquals("new-content", cache.get(Instant.ofEpochMilli(4))); + verifyNoMoreInteractions(unixPath); + } + +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java new file mode 100644 index 00000000000..44868e17464 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java @@ -0,0 +1,84 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Test; + +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class FileSyncTest { + private final TestTaskContext taskContext = new TestTaskContext(); + private final FileSystem fileSystem = TestFileSystem.create(); + + private final Path path = fileSystem.getPath("/dir/file.txt"); + private final UnixPath unixPath = new UnixPath(path); + private final FileSync fileSync = new FileSync(path); + + private String content = "content"; + private String owner = "owner"; // default is user + private String group = "group1"; // default is group + private String permissions = "rw-r-xr--"; + + @Test + public void trivial() { + assertConvergence("Creating file /dir/file.txt", + "Changing owner of /dir/file.txt from user to owner", + "Changing group of /dir/file.txt from group to group1", + "Changing permissions of /dir/file.txt from rw-r--r-- to rw-r-xr--"); + + content = "new-content"; + assertConvergence("Patching file /dir/file.txt"); + + owner = "new-owner"; + assertConvergence("Changing owner of /dir/file.txt from owner to " + + owner); + + group = "new-group1"; + assertConvergence("Changing group of /dir/file.txt from group1 to new-group1"); + + permissions = "rwxr--rwx"; + assertConvergence("Changing permissions of /dir/file.txt from rw-r-xr-- to " + + permissions); + } + + private void assertConvergence(String... systemModificationMessages) { + PartialFileData fileData = PartialFileData.builder() + .withContent(content) + .withOwner(owner) + .withGroup(group) + .withPermissions(permissions) + .create(); + taskContext.clearSystemModificationLog(); + assertTrue(fileSync.convergeTo(taskContext, fileData)); + + assertTrue(Files.isRegularFile(path)); + fileData.getContent().ifPresent(content -> assertEquals(content, unixPath.readUtf8File())); + fileData.getOwner().ifPresent(owner -> assertEquals(owner, unixPath.getOwner())); + fileData.getGroup().ifPresent(group -> assertEquals(group, unixPath.getGroup())); + fileData.getPermissions().ifPresent(permissions -> assertEquals(permissions, unixPath.getPermissions())); + + List<String> actualMods = taskContext.getSystemModificationLog(); + List<String> expectedMods = Arrays.asList(systemModificationMessages); + assertEquals(expectedMods, actualMods); + + UnixPath unixPath = new UnixPath(path); + Instant lastModifiedTime = unixPath.getLastModifiedTime(); + taskContext.clearSystemModificationLog(); + assertFalse(fileSync.convergeTo(taskContext, fileData)); + assertEquals(lastModifiedTime, unixPath.getLastModifiedTime()); + + actualMods = taskContext.getSystemModificationLog(); + assertEquals(new ArrayList<>(), actualMods); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java index ca4eabf855b..bb8ca2586c8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; +import com.yahoo.vespa.test.file.TestFileSystem; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import org.junit.Test; @@ -32,10 +33,11 @@ public class FileWriterTest { FileWriter writer = new FileWriter(path, () -> content) .withPermissions(permissions) .withOwner(owner) - .withGroup(group); + .withGroup(group) + .onlyIfFileDoesNotAlreadyExist(); TaskContext context = mock(TaskContext.class); assertTrue(writer.converge(context)); - verify(context, times(1)).logSystemModification(any(), eq("Writing file " + path)); + verify(context, times(1)).logSystemModification(any(), eq("Creating file " + path)); UnixPath unixPath = new UnixPath(path); assertEquals(content, unixPath.readUtf8File()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFileTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFileTest.java new file mode 100644 index 00000000000..b1d88fdaaee --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFileTest.java @@ -0,0 +1,49 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * WARNING: Velocity does not honor an alternative FileSystem like JimFS. + */ +public class TemplateFileTest { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + private void writeFile(Path path, String content) { + UnixPath unixPath = new UnixPath(path); + unixPath.createParents(); + unixPath.writeUtf8File(content); + } + + @Test + public void basic() throws IOException { + String templateContent = "a $x, $y b"; + Path templatePath = folder.newFile("example.vm").toPath(); + writeFile(templatePath, templateContent); + + Path toPath = folder.newFile().toPath(); + TaskContext taskContext = mock(TaskContext.class); + boolean converged = new TemplateFile(templatePath) + .set("x", "foo") + .set("y", "bar") + .getFileWriterTo(toPath) + .converge(taskContext); + + assertTrue(converged); + + String actualContent = new UnixPath(toPath).readUtf8File(); + assertEquals("a foo, bar b", actualContent); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestFileSystem.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestFileSystem.java deleted file mode 100644 index 465cb671a97..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestFileSystem.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.google.common.jimfs.Configuration; -import com.google.common.jimfs.Feature; -import com.google.common.jimfs.Jimfs; -import com.google.common.jimfs.PathType; - -import java.nio.file.FileSystem; - -public class TestFileSystem { - public static FileSystem create() { - // This configuration is based on Configuration.unix(), except: - // - Use "posix" attribute view which is necessary for permissions, owner, and group. - Configuration configuration = Configuration.builder(PathType.unix()) - .setRoots("/") - .setWorkingDirectory("/work") - .setAttributeViews("posix") - .setSupportedFeatures(Feature.LINKS, Feature.SYMBOLIC_LINKS, Feature.SECURE_DIRECTORY_STREAM, Feature.FILE_CHANNEL) - .build(); - return Jimfs.newFileSystem(configuration); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java new file mode 100644 index 00000000000..757f3004683 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java @@ -0,0 +1,49 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.IdempotentTask; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.nio.file.FileSystem; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.logging.Logger; + +public class TestTaskContext implements TaskContext { + private final List<String> logs = new ArrayList<>(); + + @Override + public Cloud cloud() { + throw new UnsupportedOperationException(); + } + + @Override + public EnumSet<Role> roles() { + throw new UnsupportedOperationException(); + } + + @Override + public FileSystem fileSystem() { + throw new UnsupportedOperationException(); + } + + @Override + public void logSystemModification(Logger logger, String actionDescription) { + logs.add(actionDescription); + } + + public List<String> getSystemModificationLog() { + return logs; + } + + public void clearSystemModificationLog() { + logs.clear(); + } + + @Override + public boolean executeSubtask(IdempotentTask task) { + throw new UnsupportedOperationException(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java index 821c6397ee7..bd29f239e1d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; +import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; import java.nio.file.FileSystem; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandTest.java new file mode 100644 index 00000000000..373c75eba59 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandTest.java @@ -0,0 +1,87 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import org.junit.Test; + +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +public class CommandTest { + @Test + public void testCommand() { + TaskContext taskContext = mock(TaskContext.class); + Logger logger = mock(Logger.class); + + Command command = new Command(taskContext).add("bash", "-c", "ls /bin/bash"); + Path outputFile; + // Assumes /bin/bash exists on all hosts running this test. + try (ChildProcess childProcess = command.spawn(logger)) { + verify(taskContext).logSystemModification(eq(logger), any()); + + outputFile = childProcess.getProcessOutputPath(); + int exitValue = childProcess.waitForTermination().exitValue(); + assertEquals(0, exitValue); + childProcess.throwIfFailed(); + String output = childProcess.getUtf8Output().trim(); + assertEquals("/bin/bash", output); + assertTrue(outputFile.toFile().exists()); + } + + assertFalse(outputFile.toFile().exists()); + } + + @Test(expected = UncheckedIOException.class) + public void noSuchProgram() { + TaskContext taskContext = mock(TaskContext.class); + Logger logger = mock(Logger.class); + + Command command = new Command(taskContext).add("thisprogRamDoes-not-exist"); + try (ChildProcess childProcess = command.spawn(logger)) { + dummyToRemoveWarning(childProcess); + } + + fail(); + } + + private void dummyToRemoveWarning(ChildProcess childProcess) { } + + @Test + public void argumentEscape() { + TaskContext taskContext = mock(TaskContext.class); + Command command = new Command(taskContext).add("b", "\" \\ foo", "bar x", ""); + assertEquals("b \"\\\" \\\\ foo\" \"bar x\" \"\"", command.commandLine()); + } + + @Test + public void failingProgram() { + TaskContext taskContext = mock(TaskContext.class); + Logger logger = mock(Logger.class); + + Command command = new Command(taskContext) + .add("bash", "-c", "echo foo; echo bar >&2; exit 1"); + Path outputFile; + try (ChildProcess childProcess = command.spawn(logger)) { + try { + childProcess.waitForTermination().throwIfFailed(); + fail(); + } catch (CommandException e) { + assertEquals("Execution of program [bash -c \"echo foo; echo bar >&2; exit 1\"] failed, stdout/stderr was: <foo\n" + + "bar\n" + + ">", + e.getMessage()); + } + } + + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommand.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommand.java new file mode 100644 index 00000000000..59c853f949d --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommand.java @@ -0,0 +1,71 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.nio.file.Path; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class TestCommand extends Command { + private final String expectedCommandLine; + private final ChildProcess childProcess; + + private boolean invoked = false; + + public TestCommand(TaskContext context, + String expectedCommandLine, + int exitValue, + String out) { + super(context); + this.expectedCommandLine = expectedCommandLine; + this.childProcess = new ChildProcess() { + @Override + public ChildProcess waitForTermination() { + return this; + } + + @Override + public int exitValue() { + return exitValue; + } + + @Override + public ChildProcess throwIfFailed() { + if (exitValue != 0) { + throw new CommandException("exited with " + exitValue); + } + return this; + } + + @Override + public String getUtf8Output() { + return out; + } + + @Override + public void close() { } + + @Override + public Path getProcessOutputPath() { return null; } + }; + } + + @Override + public ChildProcess spawn(Logger commandLogger) { + assertFalse(invoked); + invoked = true; + + assertEquals(expectedCommandLine, commandLine()); + + return childProcess; + } + + public void verifyInvocation() { + if (!invoked) { + throw new IllegalStateException("Command not invoked: " + expectedCommandLine); + } + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommandSupplier.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommandSupplier.java new file mode 100644 index 00000000000..1c900604260 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestCommandSupplier.java @@ -0,0 +1,43 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class TestCommandSupplier implements Supplier<Command> { + private final TaskContext taskContext; + private final List<TestCommand> expectedInvocations = new ArrayList<>(); + private int index = 0; + + public TestCommandSupplier(TaskContext taskContext) { + this.taskContext = taskContext; + } + + public TestCommandSupplier expectCommand(String commandLine, int exitValue, String out) { + expectedInvocations.add(new TestCommand(taskContext, commandLine, exitValue, out)); + return this; + } + + @Override + public Command get() { + if (index >= expectedInvocations.size()) { + throw new IllegalStateException("Too many command invocations"); + } + + return expectedInvocations.get(index++); + } + + public void verifyInvocations() { + if (index != expectedInvocations.size()) { + throw new IllegalStateException("Received only " + index + + " command invocations: expected " + expectedInvocations.size()); + } + + for (int i = 0; i < index; ++i) { + expectedInvocations.get(i).verifyInvocation(); + } + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepoTest.java index 7b6ab91345b..ad1fefe782f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepoTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.yum; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.TestFileSystem; +import com.yahoo.vespa.test.file.TestFileSystem; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import org.junit.Test; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java new file mode 100644 index 00000000000..d852be26229 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java @@ -0,0 +1,67 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.yum; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandException; +import com.yahoo.vespa.hosted.node.admin.task.util.process.TestCommandSupplier; +import org.junit.Test; + +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +public class YumTest { + @Test + public void testAlreadyInstalled() { + TaskContext taskContext = mock(TaskContext.class); + TestCommandSupplier commandSupplier = new TestCommandSupplier(taskContext); + + commandSupplier.expectCommand("yum list installed package-1", 0, ""); + commandSupplier.expectCommand("yum list installed package-2", 0, ""); + + Yum yum = new Yum(taskContext, commandSupplier); + yum.install("package-1", "package-2") + .enableRepo("repo-name") + .converge(); + + commandSupplier.verifyInvocations(); + } + + @Test + public void testInstall() { + TaskContext taskContext = mock(TaskContext.class); + TestCommandSupplier commandSupplier = new TestCommandSupplier(taskContext); + + commandSupplier.expectCommand("yum list installed package-1", 0, ""); + commandSupplier.expectCommand("yum list installed package-2", 1, ""); + commandSupplier.expectCommand( + "yum install --assumeyes --enablerepo=repo-name package-1 package-2", + 0, + ""); + + Yum yum = new Yum(taskContext, commandSupplier); + yum.install("package-1", "package-2") + .enableRepo("repo-name") + .converge(); + + commandSupplier.verifyInvocations(); + } + + @Test(expected = CommandException.class) + public void testFailedInstall() { + TaskContext taskContext = mock(TaskContext.class); + TestCommandSupplier commandSupplier = new TestCommandSupplier(taskContext); + + commandSupplier.expectCommand("yum list installed package-1", 0, ""); + commandSupplier.expectCommand("yum list installed package-2", 1, ""); + commandSupplier.expectCommand( + "yum install --assumeyes --enablerepo=repo-name package-1 package-2", + 1, + "error"); + + Yum yum = new Yum(taskContext, commandSupplier); + yum.install("package-1", "package-2") + .enableRepo("repo-name") + .converge(); + fail(); + } +}
\ No newline at end of file |