diff options
author | Håkon Hallingstad <hakon@verizonmedia.com> | 2019-09-27 15:40:30 +0200 |
---|---|---|
committer | Håkon Hallingstad <hakon@verizonmedia.com> | 2019-09-27 15:40:30 +0200 |
commit | 6f27ed294b3c4ca74ccbf6c9d7d41710015c4917 (patch) | |
tree | 03231f5767c0ab3b721ad2a67170f36612c28e5d /node-admin | |
parent | 4d00bb40718ab4e01230e1492d73a2d92e0124f9 (diff) |
Make FileSnapshot for repeated reads
Diffstat (limited to 'node-admin')
4 files changed, 171 insertions, 2 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java new file mode 100644 index 00000000000..4f227ccb6d4 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java @@ -0,0 +1,83 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; + +/** + * A snapshot of the attributes of the file for a given path, and file content if it is a regular file. + * + * @author hakonhall + */ +public class FileSnapshot { + private final Path path; + private final Optional<FileAttributes> attributes; + private final Optional<byte[]> content; + + public static FileSnapshot forPath(Path path) { return forNonExistingFile(path).snapshot(); } + + /** Guaranteed to not throw any exceptions. */ + public static FileSnapshot forNonExistingFile(Path path) { + return new FileSnapshot(path, Optional.empty(), Optional.empty()); + } + + private static FileSnapshot forRegularFile(Path path, FileAttributes attributes, byte[] content) { + if (!attributes.isRegularFile()) throw new IllegalArgumentException(path + " is not a regular file"); + return new FileSnapshot(path, Optional.of(attributes), Optional.of(content)); + } + + private static FileSnapshot forOtherFile(Path path, FileAttributes attributes) { + if (attributes.isRegularFile()) throw new IllegalArgumentException(path + " is a regular file"); + return new FileSnapshot(path, Optional.of(attributes), Optional.empty()); + } + + private FileSnapshot(Path path, Optional<FileAttributes> attributes, Optional<byte[]> content) { + this.path = path; + this.attributes = attributes; + this.content = content; + } + + public Path path() { return path; } + + /** Whether there was a file (or directory) at path. */ + public boolean exists() { return attributes.isPresent(); } + + /** Returns the file attributes if the file exists. */ + public Optional<FileAttributes> attributes() { return attributes; } + + /** Returns the file content if the file exists and is a regular file. */ + public Optional<byte[]> content() { return content; } + + /** Returns the file UTF-8 content if it exists and is a regular file. */ + public Optional<String> utf8Content() { return content.map(c -> new String(c, StandardCharsets.UTF_8)); } + + /** Returns an up-to-date snapshot of the path, possibly {@code this} if last modified time has not changed. */ + public FileSnapshot snapshot() { + Optional<FileAttributes> currentAttributes = new UnixPath(path).getAttributesIfExists(); + if (currentAttributes.isPresent()) { + + // 'this' may still be valid, depending on last modified times. + if (attributes.isPresent()) { + Instant previousModifiedTime = attributes.get().lastModifiedTime(); + Instant currentModifiedTime = currentAttributes.get().lastModifiedTime(); + if (currentModifiedTime.compareTo(previousModifiedTime) <= 0) { + return this; + } + } + + if (currentAttributes.get().isRegularFile()) { + Optional<byte[]> content = IOExceptionUtil.ifExists(() -> Files.readAllBytes(path)); + return content.map(bytes -> FileSnapshot.forRegularFile(path, currentAttributes.get(), bytes)) + // File was removed after getting attributes and before getting content. + .orElseGet(() -> FileSnapshot.forNonExistingFile(path)); + } else { + return FileSnapshot.forOtherFile(path, currentAttributes.get()); + } + } else { + return attributes.isPresent() ? FileSnapshot.forNonExistingFile(path) : this /* avoid allocation */; + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java index f7eba68e455..afc0e7b5c22 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java @@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; import java.util.function.Supplier; /** @@ -17,20 +19,30 @@ public class FileWriter { private final Path path; private final FileSync fileSync; private final PartialFileData.Builder fileDataBuilder = PartialFileData.builder(); - private final Supplier<byte[]> contentProducer; + private final Optional<ByteArraySupplier> contentProducer; private boolean overwriteExistingFile = true; + public FileWriter(Path path) { + this(path, Optional.empty()); + } + public FileWriter(Path path, Supplier<String> contentProducer) { this(path, () -> contentProducer.get().getBytes(StandardCharsets.UTF_8)); } public FileWriter(Path path, ByteArraySupplier contentProducer) { + this(path, Optional.of(contentProducer)); + } + + private FileWriter(Path path, Optional<ByteArraySupplier> contentProducer) { this.path = path; this.fileSync = new FileSync(path); this.contentProducer = contentProducer; } + public Path path() { return path; } + public FileWriter withOwner(String owner) { fileDataBuilder.withOwner(owner); return this; @@ -52,11 +64,19 @@ public class FileWriter { } public boolean converge(TaskContext context) { + return converge(context, contentProducer.orElseThrow().get()); + } + + public boolean converge(TaskContext context, String utf8Content) { + return converge(context, utf8Content.getBytes(StandardCharsets.UTF_8)); + } + + public boolean converge(TaskContext context, byte[] content) { if (!overwriteExistingFile && Files.isRegularFile(path)) { return false; } - fileDataBuilder.withContent(contentProducer.get()); + fileDataBuilder.withContent(content); PartialFileData fileData = fileDataBuilder.create(); return fileSync.convergeTo(context, fileData); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index cf6c6c432f4..2cc74742463 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -231,6 +231,8 @@ public class UnixPath { return new UnixPath(link); } + public FileSnapshot getFileSnapshot() { return FileSnapshot.forPath(path); } + @Override public String toString() { return path.toString(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java new file mode 100644 index 00000000000..8c73d522f1d --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java @@ -0,0 +1,64 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Test; + +import java.nio.file.FileSystem; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author hakonhall + */ +public class FileSnapshotTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final UnixPath path = new UnixPath(fileSystem.getPath("/var/lib/file.txt")); + + private FileSnapshot fileSnapshot = FileSnapshot.forPath(path.toPath()); + + @Test + public void fileDoesNotExist() { + assertFalse(fileSnapshot.exists()); + assertFalse(fileSnapshot.attributes().isPresent()); + assertFalse(fileSnapshot.content().isPresent()); + assertEquals(path.toPath(), fileSnapshot.path()); + } + + @Test + public void directory() { + path.createParents().createDirectory(); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + assertTrue(fileSnapshot.attributes().isPresent()); + assertTrue(fileSnapshot.attributes().get().isDirectory()); + } + + @Test + public void regularFile() { + path.createParents().writeUtf8File("file content"); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + assertTrue(fileSnapshot.attributes().isPresent()); + assertTrue(fileSnapshot.attributes().get().isRegularFile()); + assertTrue(fileSnapshot.utf8Content().isPresent()); + assertEquals("file content", fileSnapshot.utf8Content().get()); + + FileSnapshot newFileSnapshot = fileSnapshot.snapshot(); + assertSame(fileSnapshot, newFileSnapshot); + } + + @Test + public void fileRemoval() { + path.createParents().writeUtf8File("file content"); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + path.deleteIfExists(); + fileSnapshot = fileSnapshot.snapshot(); + assertFalse(fileSnapshot.exists()); + } +}
\ No newline at end of file |