From c3168ca58af1396e7a7562fdeb15fa604425020d Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sat, 27 Jan 2018 02:54:18 +0100 Subject: Support file templates --- node-admin/pom.xml | 6 +- .../node/admin/task/util/file/FileAttributes.java | 22 ++++ .../admin/task/util/file/FileAttributesCache.java | 44 ++++++++ .../admin/task/util/file/FileContentCache.java | 34 ++++++ .../hosted/node/admin/task/util/file/FileSync.java | 116 +++++++++++++++++++++ .../node/admin/task/util/file/FileWriter.java | 40 +++---- .../node/admin/task/util/file/IOExceptionUtil.java | 18 ++++ .../node/admin/task/util/file/PartialFileData.java | 62 +++++++++++ .../node/admin/task/util/file/TemplateFile.java | 42 ++++++++ .../hosted/node/admin/task/util/file/UnixPath.java | 59 +++++------ .../node/admin/task/util/yum/AddYumRepo.java | 3 +- .../task/util/file/FileAttributesCacheTest.java | 38 +++++++ .../admin/task/util/file/FileContentCacheTest.java | 58 +++++++++++ .../node/admin/task/util/file/FileSyncTest.java | 83 +++++++++++++++ .../node/admin/task/util/file/FileWriterTest.java | 5 +- .../admin/task/util/file/TemplateFileTest.java | 49 +++++++++ .../node/admin/task/util/file/TestTaskContext.java | 45 ++++++++ parent/pom.xml | 5 + .../com/yahoo/vespa/test/file/TestFileSystem.java | 2 + 19 files changed, 674 insertions(+), 57 deletions(-) create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFile.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFileTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 983e4d3a832..161769a4edf 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -115,8 +115,12 @@ mockito-core test + + org.apache.velocity + velocity + compile + - diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java new file mode 100644 index 00000000000..c99d5850909 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java @@ -0,0 +1,22 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Instant; + +public class FileAttributes { + private final PosixFileAttributes attributes; + + FileAttributes(PosixFileAttributes attributes) { + this.attributes = attributes; + } + + public Instant lastModifiedTime() { return attributes.lastModifiedTime().toInstant(); } + public String owner() { return attributes.owner().getName(); } + public String group() { return attributes.group().getName(); } + public String permissions() { return PosixFilePermissions.toString(attributes.permissions()); } + public boolean isRegularFile() { return attributes.isRegularFile(); } + public boolean isDirectory() { return attributes.isDirectory(); } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java new file mode 100644 index 00000000000..12a9609f89c --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java @@ -0,0 +1,44 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.util.Optional; + +// @ThreadUnsafe +public class FileAttributesCache { + private final UnixPath path; + + private Optional attributes = Optional.empty(); + + public FileAttributesCache(UnixPath path) { + this.path = path; + } + + public FileAttributes get() { + if (!attributes.isPresent()) { + attributes = Optional.of(path.getAttributes()); + } + + return attributes.get(); + } + + public FileAttributes forceGet() { + attributes = Optional.empty(); + return get(); + } + + public boolean exists() { + if (attributes.isPresent()) { + return true; + } + + Optional attributes = path.getAttributesIfExists(); + if (attributes.isPresent()) { + // Might as well update this.attributes + this.attributes = attributes; + return true; + } else { + return false; + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java new file mode 100644 index 00000000000..ac50ff0cbab --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java @@ -0,0 +1,34 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.time.Instant; +import java.util.Optional; + +/** + * Class to avoid repeated reads of file content when the file seldom changes. + */ +class FileContentCache { + private final UnixPath path; + + private Optional value = Optional.empty(); + private Optional modifiedTime = Optional.empty(); + + FileContentCache(UnixPath path) { + this.path = path; + } + + String get(Instant lastModifiedTime) { + if (!value.isPresent() || lastModifiedTime.compareTo(modifiedTime.get()) > 0) { + value = Optional.of(path.readUtf8File()); + modifiedTime = Optional.of(lastModifiedTime); + } + + return value.get(); + } + + void updateWith(String content, Instant modifiedTime) { + this.value = Optional.of(content); + this.modifiedTime = Optional.of(modifiedTime); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java new file mode 100644 index 00000000000..f103ab394ef --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java @@ -0,0 +1,116 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.logging.Logger; + +/** + * Class to minimize resource usage with repetitive and mostly identical, idempotent, and + * mutating file operations, e.g. setting file content, setting owner, etc. + * + * Only changes to the file is logged. + */ +// @ThreadUnsafe +public class FileSync { + private static final Logger logger = Logger.getLogger(FileSync.class.getName()); + + private final UnixPath path; + private final FileContentCache contentCache; + + public FileSync(Path path) { + this.path = new UnixPath(path); + this.contentCache = new FileContentCache(this.path); + } + + /** + * CPU, I/O, and memory usage is optimized for repeated calls with the same arguments. + * @return true if the system was modified: content was written, or owner was set, etc. + * system is only modified if necessary (different). + */ + public boolean convergeTo(TaskContext taskContext, PartialFileData partialFileData) { + FileAttributesCache currentAttributes = new FileAttributesCache(path); + + boolean modifiedSystem = false; + + modifiedSystem |= maybeUpdateContent(taskContext, partialFileData.getContent(), currentAttributes); + + modifiedSystem |= convergeAttribute( + taskContext, + "owner", + partialFileData.getOwner(), + () -> currentAttributes.get().owner(), + path::setOwner); + + modifiedSystem |= convergeAttribute( + taskContext, + "group", + partialFileData.getGroup(), + () -> currentAttributes.get().group(), + path::setGroup); + + modifiedSystem |= convergeAttribute( + taskContext, + "permissions", + partialFileData.getPermissions(), + () -> currentAttributes.get().permissions(), + path::setPermissions); + + return modifiedSystem; + } + + private boolean convergeAttribute(TaskContext taskContext, + String attributeName, + Optional wantedValue, + Supplier currentValueSupplier, + Consumer valueSetter) { + if (!wantedValue.isPresent()) { + return false; + } + + String currentValue = currentValueSupplier.get(); + if (Objects.equals(wantedValue.get(), currentValue)) { + return false; + } else { + String actionDescription = String.format("Changing %s of %s from %s to %s", + attributeName, + path, + currentValue, + wantedValue.get()); + taskContext.logSystemModification(logger, actionDescription); + valueSetter.accept(wantedValue.get()); + return true; + } + } + + private boolean maybeUpdateContent(TaskContext taskContext, + Optional content, + FileAttributesCache currentAttributes) { + if (!content.isPresent()) { + return false; + } + + if (!currentAttributes.exists()) { + taskContext.logSystemModification(logger, "Creating file " + path); + path.createParents(); + path.writeUtf8File(content.get()); + contentCache.updateWith(content.get(), currentAttributes.forceGet().lastModifiedTime()); + return true; + } + + if (Objects.equals(content.get(), contentCache.get(currentAttributes.get().lastModifiedTime()))) { + return false; + } else { + taskContext.logSystemModification(logger, "Patching file " + path); + path.writeUtf8File(content.get()); + contentCache.updateWith(content.get(), currentAttributes.forceGet().lastModifiedTime()); + return true; + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java index 60a7b3482b2..2f22c94781f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java @@ -6,56 +6,48 @@ import org.glassfish.jersey.internal.util.Producer; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Optional; -import java.util.logging.Logger; public class FileWriter { - private static final Logger logger = Logger.getLogger(FileWriter.class.getName()); - private final Path path; + private final FileSync fileSync; + private final PartialFileData.Builder fileDataBuilder = PartialFileData.builder(); private final Producer contentProducer; - private Optional owner = Optional.empty(); - private Optional group = Optional.empty(); - private Optional permissions = Optional.empty(); + private boolean overwriteExistingFile = true; public FileWriter(Path path, Producer contentProducer) { this.path = path; + this.fileSync = new FileSync(path); this.contentProducer = contentProducer; } public FileWriter withOwner(String owner) { - this.owner = Optional.of(owner); + fileDataBuilder.withOwner(owner); return this; } public FileWriter withGroup(String group) { - this.group = Optional.of(group); + fileDataBuilder.withGroup(group); return this; } public FileWriter withPermissions(String permissions) { - this.permissions = Optional.of(permissions); + fileDataBuilder.withPermissions(permissions); + return this; + } + + public FileWriter onlyIfFileDoesNotAlreadyExist() { + overwriteExistingFile = false; return this; } public boolean converge(TaskContext context) { - // TODO: Only return false if content, permission, etc would be unchanged. - if (Files.isRegularFile(path)) { + if (!overwriteExistingFile && Files.isRegularFile(path)) { return false; } - context.logSystemModification(logger,"Writing file " + path); - - String content = contentProducer.call(); - - UnixPath unixPath = new UnixPath(path); - unixPath.createParents(); - unixPath.writeUtf8File(content); - permissions.ifPresent(unixPath::setPermissions); - owner.ifPresent(unixPath::setOwner); - group.ifPresent(unixPath::setGroup); - - return true; + fileDataBuilder.withContent(contentProducer.call()); + PartialFileData fileData = fileDataBuilder.create(); + return fileSync.convergeTo(context, fileData); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java index dee5525d42a..ef553a131e8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java @@ -1,8 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.file; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Optional; public class IOExceptionUtil { public static void uncheck(RunnableThrowingIOException runnable) { @@ -31,4 +33,20 @@ public class IOExceptionUtil { public interface RunnableThrowingIOException { void run() throws IOException; } + + /** + * Useful if it's not known whether a file or directory exists, in case e.g. + * FileNotFoundException is thrown and the caller wants an Optional.empty() in that case. + */ + public static Optional ifExists(SupplierThrowingIOException supplier) { + try { + return Optional.ofNullable(uncheck(supplier)); + } catch (UncheckedIOException e) { + if (e.getCause() instanceof FileNotFoundException) { + return Optional.empty(); + } + + throw e; + } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java new file mode 100644 index 00000000000..d29c01179f2 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java @@ -0,0 +1,62 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.util.Optional; + +/** + * Represents a subset of a file's content, owner, group, and permissions. + */ +// @Immutable +public class PartialFileData { + private final Optional content; + private final Optional owner; + private final Optional group; + private final Optional permissions; + + public static Builder builder() { + return new Builder(); + } + + public PartialFileData(Optional content, + Optional owner, + Optional group, + Optional permissions) { + this.content = content; + this.owner = owner; + this.group = group; + this.permissions = permissions; + } + + public Optional getContent() { + return content; + } + + public Optional getOwner() { + return owner; + } + + public Optional getGroup() { + return group; + } + + public Optional getPermissions() { + return permissions; + } + + public static class Builder { + private Optional content = Optional.empty(); + private Optional owner = Optional.empty(); + private Optional group = Optional.empty(); + private Optional permissions = Optional.empty(); + + public Builder withContent(String content) { this.content = Optional.of(content); return this; } + public Builder withOwner(String owner) { this.owner = Optional.of(owner); return this; } + public Builder withGroup(String group) { this.group = Optional.of(group); return this; } + public Builder withPermissions(String permissions) { this.permissions = Optional.of(permissions); return this; } + + public PartialFileData create() { + return new PartialFileData(content, owner, group, permissions); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFile.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFile.java new file mode 100644 index 00000000000..1d5b5f42cf3 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFile.java @@ -0,0 +1,42 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.apache.velocity.Template; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; +import org.apache.velocity.app.VelocityEngine; + +import java.io.StringWriter; +import java.nio.file.Path; + +public class TemplateFile { + private final Path templatePath; + private final VelocityEngine velocityEngine; + private final VelocityContext velocityContext = new VelocityContext(); + + public TemplateFile(Path templatePath) { + this.templatePath = templatePath; + velocityEngine = new VelocityEngine(); + velocityEngine.addProperty( + Velocity.RUNTIME_LOG_LOGSYSTEM_CLASS, + "org.apache.velocity.runtime.log.NullLogSystem"); + velocityEngine.addProperty(Velocity.FILE_RESOURCE_LOADER_PATH, templatePath.getParent().toString()); + velocityEngine.init(); + } + + public TemplateFile set(String name, String value) { + velocityContext.put(name, value); + return this; + } + + public FileWriter getFileWriterTo(Path destinationPath) { + return new FileWriter(destinationPath, this::render); + } + + private String render() { + Template template = velocityEngine.getTemplate(templatePath.getFileName().toString(), "UTF-8"); + StringWriter writer = new StringWriter(); + template.merge(velocityContext, writer); + return writer.toString(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 606f8cfb06e..a83d79a97a9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -1,10 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.file; -import java.io.IOException; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.Paths; @@ -16,8 +16,11 @@ import java.nio.file.attribute.PosixFilePermissions; import java.nio.file.attribute.UserPrincipal; import java.nio.file.attribute.UserPrincipalLookupService; import java.time.Instant; +import java.util.Optional; import java.util.Set; +import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck; + // @Immutable public class UnixPath { private final Path path; @@ -34,8 +37,14 @@ public class UnixPath { return path; } - public void createParents() { - uncheck(() -> Files.createDirectories(path.getParent())); + public boolean createParents() { + Path parent = path.getParent(); + if (Files.isDirectory(parent)) { + return false; + } + + uncheck(() -> Files.createDirectories(parent)); + return true; } public String readUtf8File() { @@ -49,7 +58,7 @@ public class UnixPath { } public String getPermissions() { - return PosixFilePermissions.toString(getAttributes().permissions()); + return getAttributes().permissions(); } /** @@ -69,7 +78,7 @@ public class UnixPath { } public String getOwner() { - return getAttributes().owner().getName(); + return getAttributes().owner(); } public void setOwner(String owner) { @@ -79,7 +88,7 @@ public class UnixPath { } public String getGroup() { - return getAttributes().group().getName(); + return getAttributes().group(); } public void setGroup(String group) { @@ -89,37 +98,29 @@ public class UnixPath { } public Instant getLastModifiedTime() { - return uncheck(() -> Files.getLastModifiedTime(path)).toInstant(); + return getAttributes().lastModifiedTime(); } - private PosixFileAttributes getAttributes() { - return uncheck(() -> + public FileAttributes getAttributes() { + PosixFileAttributes attributes = uncheck(() -> Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes()); + return new FileAttributes(attributes); } - @FunctionalInterface - private interface SupplierThrowingIOException { - T get() throws IOException; - } - - private static T uncheck(SupplierThrowingIOException supplier) { + public Optional getAttributesIfExists() { try { - return supplier.get(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } + return Optional.of(getAttributes()); + } catch (UncheckedIOException e) { + if (e.getCause() instanceof NoSuchFileException) { + return Optional.empty(); + } - @FunctionalInterface - private interface RunnableThrowingIOException { - void run() throws IOException; + throw e; + } } - private static void uncheck(RunnableThrowingIOException runnable) { - try { - runnable.run(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + @Override + public String toString() { + return path.toString(); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java index 9ca1c0286f9..d6dbc3cd12a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java @@ -32,7 +32,8 @@ public class AddYumRepo { FileWriter fileWriter = new FileWriter(path, this::getRepoFileContent) .withOwner("root") .withGroup("root") - .withPermissions("rw-r--r--"); + .withPermissions("rw-r--r--") + .onlyIfFileDoesNotAlreadyExist(); return fileWriter.converge(context); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java new file mode 100644 index 00000000000..9224faf1c6f --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java @@ -0,0 +1,38 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class FileAttributesCacheTest { + @Test + public void exists() throws Exception { + UnixPath unixPath = mock(UnixPath.class); + FileAttributesCache cache = new FileAttributesCache(unixPath); + + when(unixPath.getAttributesIfExists()).thenReturn(Optional.empty()); + assertFalse(cache.exists()); + verify(unixPath, times(1)).getAttributesIfExists(); + verifyNoMoreInteractions(unixPath); + + FileAttributes attributes = mock(FileAttributes.class); + when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes)); + assertTrue(cache.exists()); + verify(unixPath, times(1 + 1)).getAttributesIfExists(); + verifyNoMoreInteractions(unixPath); + + assertEquals(attributes, cache.get()); + verifyNoMoreInteractions(unixPath); + } +} \ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java new file mode 100644 index 00000000000..677dd048445 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java @@ -0,0 +1,58 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.Test; + +import java.time.Instant; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class FileContentCacheTest { + private final UnixPath unixPath = mock(UnixPath.class); + private final FileContentCache cache = new FileContentCache(unixPath); + + @Test + public void get() throws Exception { + when(unixPath.readUtf8File()).thenReturn("content"); + assertEquals("content", cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1)).readUtf8File(); + verifyNoMoreInteractions(unixPath); + + // cache hit + assertEquals("content", cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1)).readUtf8File(); + verifyNoMoreInteractions(unixPath); + + // cache miss + when(unixPath.readUtf8File()).thenReturn("new-content"); + assertEquals("new-content", cache.get(Instant.ofEpochMilli(1))); + verify(unixPath, times(1 + 1)).readUtf8File(); + verifyNoMoreInteractions(unixPath); + + // cache hit both at times 0 and 1 + assertEquals("new-content", cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1 + 1)).readUtf8File(); + verifyNoMoreInteractions(unixPath); + assertEquals("new-content", cache.get(Instant.ofEpochMilli(1))); + verify(unixPath, times(1 + 1)).readUtf8File(); + verifyNoMoreInteractions(unixPath); + } + + @Test + public void updateWith() throws Exception { + cache.updateWith("content", Instant.ofEpochMilli(2)); + assertEquals("content", cache.get(Instant.ofEpochMilli(2))); + verifyNoMoreInteractions(unixPath); + + cache.updateWith("new-content", Instant.ofEpochMilli(4)); + assertEquals("new-content", cache.get(Instant.ofEpochMilli(4))); + verifyNoMoreInteractions(unixPath); + } + +} \ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java new file mode 100644 index 00000000000..71cd312d5d6 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java @@ -0,0 +1,83 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.Test; + +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class FileSyncTest { + private final TestTaskContext taskContext = new TestTaskContext(); + private final FileSystem fileSystem = taskContext.fileSystem(); + + private final Path path = fileSystem.getPath("/dir/file.txt"); + private final UnixPath unixPath = new UnixPath(path); + private final FileSync fileSync = new FileSync(path); + + private String content = "content"; + private String owner = "owner"; // default is user + private String group = "group1"; // default is group + private String permissions = "rw-r-xr--"; + + @Test + public void trivial() { + assertConvergence("Creating file /dir/file.txt", + "Changing owner of /dir/file.txt from user to owner", + "Changing group of /dir/file.txt from group to group1", + "Changing permissions of /dir/file.txt from rw-r--r-- to rw-r-xr--"); + + content = "new-content"; + assertConvergence("Patching file /dir/file.txt"); + + owner = "new-owner"; + assertConvergence("Changing owner of /dir/file.txt from owner to " + + owner); + + group = "new-group1"; + assertConvergence("Changing group of /dir/file.txt from group1 to new-group1"); + + permissions = "rwxr--rwx"; + assertConvergence("Changing permissions of /dir/file.txt from rw-r-xr-- to " + + permissions); + } + + private void assertConvergence(String... systemModificationMessages) { + PartialFileData fileData = PartialFileData.builder() + .withContent(content) + .withOwner(owner) + .withGroup(group) + .withPermissions(permissions) + .create(); + taskContext.clearSystemModificationLog(); + assertTrue(fileSync.convergeTo(taskContext, fileData)); + + assertTrue(Files.isRegularFile(path)); + fileData.getContent().ifPresent(content -> assertEquals(content, unixPath.readUtf8File())); + fileData.getOwner().ifPresent(owner -> assertEquals(owner, unixPath.getOwner())); + fileData.getGroup().ifPresent(group -> assertEquals(group, unixPath.getGroup())); + fileData.getPermissions().ifPresent(permissions -> assertEquals(permissions, unixPath.getPermissions())); + + List actualMods = taskContext.getSystemModificationLog(); + List expectedMods = Arrays.asList(systemModificationMessages); + assertEquals(expectedMods, actualMods); + + UnixPath unixPath = new UnixPath(path); + Instant lastModifiedTime = unixPath.getLastModifiedTime(); + taskContext.clearSystemModificationLog(); + assertFalse(fileSync.convergeTo(taskContext, fileData)); + assertEquals(lastModifiedTime, unixPath.getLastModifiedTime()); + + actualMods = taskContext.getSystemModificationLog(); + assertEquals(new ArrayList<>(), actualMods); + } +} \ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java index 878b1e4cbdb..bb8ca2586c8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java @@ -33,10 +33,11 @@ public class FileWriterTest { FileWriter writer = new FileWriter(path, () -> content) .withPermissions(permissions) .withOwner(owner) - .withGroup(group); + .withGroup(group) + .onlyIfFileDoesNotAlreadyExist(); TaskContext context = mock(TaskContext.class); assertTrue(writer.converge(context)); - verify(context, times(1)).logSystemModification(any(), eq("Writing file " + path)); + verify(context, times(1)).logSystemModification(any(), eq("Creating file " + path)); UnixPath unixPath = new UnixPath(path); assertEquals(content, unixPath.readUtf8File()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFileTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFileTest.java new file mode 100644 index 00000000000..b1d88fdaaee --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFileTest.java @@ -0,0 +1,49 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * WARNING: Velocity does not honor an alternative FileSystem like JimFS. + */ +public class TemplateFileTest { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + private void writeFile(Path path, String content) { + UnixPath unixPath = new UnixPath(path); + unixPath.createParents(); + unixPath.writeUtf8File(content); + } + + @Test + public void basic() throws IOException { + String templateContent = "a $x, $y b"; + Path templatePath = folder.newFile("example.vm").toPath(); + writeFile(templatePath, templateContent); + + Path toPath = folder.newFile().toPath(); + TaskContext taskContext = mock(TaskContext.class); + boolean converged = new TemplateFile(templatePath) + .set("x", "foo") + .set("y", "bar") + .getFileWriterTo(toPath) + .converge(taskContext); + + assertTrue(converged); + + String actualContent = new UnixPath(toPath).readUtf8File(); + assertEquals("a foo, bar b", actualContent); + } +} \ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java new file mode 100644 index 00000000000..d023a11671e --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java @@ -0,0 +1,45 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; + +import java.nio.file.FileSystem; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.logging.Logger; + +public class TestTaskContext implements TaskContext { + private final FileSystem fileSystem = TestFileSystem.create(); + private final List logs = new ArrayList<>(); + + @Override + public Cloud cloud() { + return Cloud.YAHOO; + } + + @Override + public EnumSet roles() { + return EnumSet.of(Role.CONFIG_SERVER_DOCKER_HOST); + } + + @Override + public FileSystem fileSystem() { + return fileSystem; + } + + @Override + public void logSystemModification(Logger logger, String actionDescription) { + logs.add(actionDescription); + } + + public List getSystemModificationLog() { + return logs; + } + + public void clearSystemModificationLog() { + logs.clear(); + } +} diff --git a/parent/pom.xml b/parent/pom.xml index 58ab845578c..02282f8218d 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -385,6 +385,11 @@ commons-exec 1.3 + + org.apache.velocity + velocity + 1.7 + io.airlift airline diff --git a/testutil/src/main/java/com/yahoo/vespa/test/file/TestFileSystem.java b/testutil/src/main/java/com/yahoo/vespa/test/file/TestFileSystem.java index 751825b0c2a..1de62297e0a 100644 --- a/testutil/src/main/java/com/yahoo/vespa/test/file/TestFileSystem.java +++ b/testutil/src/main/java/com/yahoo/vespa/test/file/TestFileSystem.java @@ -21,4 +21,6 @@ public class TestFileSystem { .build(); return Jimfs.newFileSystem(configuration); } + + private TestFileSystem() { } } -- cgit v1.2.3