diff options
7 files changed, 330 insertions, 57 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java index 87491367514..cee2dc9b66b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java @@ -18,6 +18,9 @@ public interface TaskContext { FileSystem fileSystem(); void logSystemModification(Logger logger, String actionDescription); + default void logSystemModification(Logger logger, String format, String... args) { + logSystemModification(logger, String.format(format, (Object[]) args)); + } default boolean executeSubtask(IdempotentTask task) { return false; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java new file mode 100644 index 00000000000..a20d30b2bf9 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java @@ -0,0 +1,126 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.logging.Logger; + +/** + * Class to converge file/directory attributes like owner and permissions to wanted values. + * Typically used by higher abstraction layers working on files (FileSync/FileWriter) or + * directories (MakeDirectory). + * + * @author hakonhall + */ +public class AttributeSync { + private static final Logger logger = Logger.getLogger(AttributeSync.class.getName()); + + private final UnixPath path; + + private Optional<String> owner = Optional.empty(); + private Optional<String> group = Optional.empty(); + private Optional<String> permissions = Optional.empty(); + + public AttributeSync(Path path) { + this.path = new UnixPath(path); + } + + public Optional<String> getPermissions() { + return permissions; + } + + public AttributeSync withPermissions(String permissions) { + this.permissions = Optional.of(permissions); + return this; + } + + public Optional<String> getOwner() { + return owner; + } + + public AttributeSync withOwner(String owner) { + this.owner = Optional.of(owner); + return this; + } + + public Optional<String> getGroup() { + return group; + } + + public AttributeSync withGroup(String group) { + this.group = Optional.of(group); + return this; + } + + public AttributeSync with(PartialFileData fileData) { + owner = fileData.getOwner(); + group = fileData.getGroup(); + permissions = fileData.getPermissions(); + return this; + } + + public boolean converge(TaskContext context) { + return converge(context, new FileAttributesCache(path)); + } + + /** + * Path must exist before calling converge. + */ + public boolean converge(TaskContext context, FileAttributesCache currentAttributes) { + boolean systemModified = updateAttribute( + context, + "owner", + owner, + () -> currentAttributes.get().owner(), + path::setOwner); + + systemModified |= updateAttribute( + context, + "group", + group, + () -> currentAttributes.get().group(), + path::setGroup); + + systemModified |= updateAttribute( + context, + "permissions", + permissions, + () -> currentAttributes.get().permissions(), + path::setPermissions); + + return systemModified; + } + + private boolean updateAttribute(TaskContext context, + String attributeName, + Optional<String> wantedValue, + Supplier<String> currentValueSupplier, + Consumer<String> valueSetter) { + if (!wantedValue.isPresent()) { + return false; + } + + String currentValue = currentValueSupplier.get(); + if (Objects.equals(currentValue, wantedValue.get())) { + return false; + } + + context.logSystemModification( + logger, + "Changing %s of %s from %s to %s", + attributeName, + path.toString(), + currentValue, + wantedValue.get()); + + valueSetter.accept(wantedValue.get()); + + return true; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java index d8b8aadfff7..18d2a2e3aa9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java @@ -7,8 +7,6 @@ import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import java.nio.file.Path; import java.util.Objects; import java.util.Optional; -import java.util.function.Consumer; -import java.util.function.Supplier; import java.util.logging.Logger; /** @@ -39,58 +37,14 @@ public class FileSync { public boolean convergeTo(TaskContext taskContext, PartialFileData partialFileData) { FileAttributesCache currentAttributes = new FileAttributesCache(path); - boolean modifiedSystem = false; + boolean modifiedSystem = maybeUpdateContent(taskContext, partialFileData.getContent(), currentAttributes); - modifiedSystem |= maybeUpdateContent(taskContext, partialFileData.getContent(), currentAttributes); - - modifiedSystem |= convergeAttribute( - taskContext, - "owner", - partialFileData.getOwner(), - () -> currentAttributes.get().owner(), - path::setOwner); - - modifiedSystem |= convergeAttribute( - taskContext, - "group", - partialFileData.getGroup(), - () -> currentAttributes.get().group(), - path::setGroup); - - modifiedSystem |= convergeAttribute( - taskContext, - "permissions", - partialFileData.getPermissions(), - () -> currentAttributes.get().permissions(), - path::setPermissions); + AttributeSync attributeSync = new AttributeSync(path.toPath()).with(partialFileData); + modifiedSystem |= attributeSync.converge(taskContext, currentAttributes); return modifiedSystem; } - private boolean convergeAttribute(TaskContext taskContext, - String attributeName, - Optional<String> wantedValue, - Supplier<String> currentValueSupplier, - Consumer<String> valueSetter) { - if (!wantedValue.isPresent()) { - return false; - } - - String currentValue = currentValueSupplier.get(); - if (Objects.equals(wantedValue.get(), currentValue)) { - return false; - } else { - String actionDescription = String.format("Changing %s of %s from %s to %s", - attributeName, - path, - currentValue, - wantedValue.get()); - taskContext.logSystemModification(logger, actionDescription); - valueSetter.accept(wantedValue.get()); - return true; - } - } - private boolean maybeUpdateContent(TaskContext taskContext, Optional<String> content, FileAttributesCache currentAttributes) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java new file mode 100644 index 00000000000..e815ab8bd86 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java @@ -0,0 +1,72 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.io.UncheckedIOException; +import java.nio.file.NotDirectoryException; +import java.nio.file.Path; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Class to ensure a directory exists with the correct owner, group, and permissions. + * + * @author hakonhall + */ +public class MakeDirectory { + private static final Logger logger = Logger.getLogger(MakeDirectory.class.getName()); + + private final UnixPath path; + private final AttributeSync attributeSync; + + private boolean createParents = false; + + public MakeDirectory(Path path) { + this.path = new UnixPath(path); + this.attributeSync = new AttributeSync(path); + } + + /** + * Warning: The owner, group, and permissions of any created parent directories are NOT modified + */ + public MakeDirectory createParents() { this.createParents = true; return this; } + + public MakeDirectory withOwner(String owner) { attributeSync.withOwner(owner); return this; } + public MakeDirectory withGroup(String group) { attributeSync.withGroup(group); return this; } + public MakeDirectory withPermissions(String permissions) { + attributeSync.withPermissions(permissions); + return this; + } + + public boolean converge(TaskContext context) { + boolean systemModified = false; + + FileAttributesCache attributes = new FileAttributesCache(path); + if (attributes.exists()) { + if (!attributes.get().isDirectory()) { + throw new UncheckedIOException(new NotDirectoryException(path.toString())); + } + } else { + if (createParents) { + // We'll skip logginer system modification here, as we'll log about the creation + // of the directory next. + path.createParents(); + } + + context.logSystemModification(logger, "Creating directory " + path); + systemModified = true; + + Optional<String> permissions = attributeSync.getPermissions(); + if (permissions.isPresent()) { + path.createDirectory(permissions.get()); + } else { + path.createDirectory(); + } + } + + systemModified |= attributeSync.converge(context, attributes); + + return systemModified; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index aaffea05d1e..ac4230ca7c6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -6,6 +6,7 @@ import java.nio.file.Files; import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.GroupPrincipal; import java.nio.file.attribute.PosixFileAttributeView; import java.nio.file.attribute.PosixFileAttributes; @@ -69,14 +70,7 @@ public class UnixPath { * and no permissions for others. */ public void setPermissions(String permissions) { - Set<PosixFilePermission> permissionSet; - try { - permissionSet = PosixFilePermissions.fromString(permissions); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Failed to set permissions '" + - permissions + "' on path " + path, e); - } - + Set<PosixFilePermission> permissionSet = getPosixFilePermissionsFromString(permissions); uncheck(() -> Files.setPosixFilePermissions(path, permissionSet)); } @@ -114,8 +108,27 @@ public class UnixPath { return IOExceptionUtil.ifExists(() -> getAttributes()); } + public void createDirectory(String permissions) { + Set<PosixFilePermission> set = getPosixFilePermissionsFromString(permissions); + FileAttribute<Set<PosixFilePermission>> attribute = PosixFilePermissions.asFileAttribute(set); + uncheck(() -> Files.createDirectory(path, attribute)); + } + + public void createDirectory() { + uncheck(() -> Files.createDirectory(path)); + } + @Override public String toString() { return path.toString(); } + + private Set<PosixFilePermission> getPosixFilePermissionsFromString(String permissions) { + try { + return PosixFilePermissions.fromString(permissions); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Failed to set permissions '" + + permissions + "' on path " + path, e); + } + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java new file mode 100644 index 00000000000..05662de3b95 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java @@ -0,0 +1,91 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Test; + +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author hakonhall + */ +public class MakeDirectoryTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final TestTaskContext context = new TestTaskContext(); + + private String path = "/parent/dir"; + private String permissions = "rwxr----x"; + private String owner = "test-owner"; + private String group = "test-group"; + + @Test + public void newDirectory() { + verifySystemModifications( + "Creating directory " + path, + "Changing owner of /parent/dir from user to test-owner", + "Changing group of /parent/dir from group to test-group"); + + owner = "new-owner"; + verifySystemModifications("Changing owner of /parent/dir from test-owner to new-owner"); + + group = "new-group"; + verifySystemModifications("Changing group of /parent/dir from test-group to new-group"); + + permissions = "--x---r--"; + verifySystemModifications("Changing permissions of /parent/dir from rwxr----x to --x---r--"); + } + + private void verifySystemModifications(String... modifications) { + context.clearSystemModificationLog(); + MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)) + .createParents() + .withPermissions(permissions) + .withOwner(owner) + .withGroup(group); + assertTrue(makeDirectory.converge(context)); + + assertEquals(Arrays.asList(modifications), context.getSystemModificationLog()); + + context.clearSystemModificationLog(); + assertFalse(makeDirectory.converge(context)); + assertEquals(Collections.emptyList(), context.getSystemModificationLog()); + } + + @Test + public void exceptionIfMissingParent() { + String path = "/parent/dir"; + MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)); + + try { + makeDirectory.converge(context); + } catch (UncheckedIOException e) { + if (e.getCause() instanceof NoSuchFileException) { + return; + } + throw e; + } + fail(); + } + + @Test + public void okIfParentExists() { + String path = "/dir"; + MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)); + assertTrue(makeDirectory.converge(context)); + assertTrue(Files.isDirectory(fileSystem.getPath(path))); + + MakeDirectory makeDirectory2 = new MakeDirectory(fileSystem.getPath(path)); + assertFalse(makeDirectory2.converge(context)); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java index bd29f239e1d..6f1991ec3d4 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java @@ -13,6 +13,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +/** + * @author hakonhall + */ public class UnixPathTest { final FileSystem fileSystem = TestFileSystem.create(); @@ -63,4 +66,15 @@ public class UnixPathTest { unixPath.setGroup("group"); assertEquals("group", unixPath.getGroup()); } + + @Test + public void createDirectoryWithPermissions() { + FileSystem fs = TestFileSystem.create(); + Path path = fs.getPath("dir"); + UnixPath unixPath = new UnixPath(path); + String permissions = "rwxr-xr--"; + unixPath.createDirectory(permissions); + assertTrue(Files.isDirectory(path)); + assertEquals(permissions, unixPath.getPermissions()); + } } |