diff options
6 files changed, 184 insertions, 2 deletions
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 54c815b48a3..54531aeb0d7 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -80,6 +80,12 @@ public class Flags { "Takes effect on next node agent tick. Change is orchestrated, but does NOT require container restart", HOSTNAME, APPLICATION_ID); + public static final UnboundBooleanFlag INCLUDE_SIS_IN_TRUSTSTORE = defineFeatureFlag( + "include-sis-in-truststore", false, + "Whether to use the trust store backed by Athenz and Service Identity certificates.", + "Takes effect on next tick, but may get throttled due to orchestration.", + HOSTNAME); + public static final UnboundStringFlag TLS_INSECURE_MIXED_MODE = defineStringFlag( "tls-insecure-mixed-mode", "tls_client_mixed_server", "TLS insecure mixed mode. Allowed values: ['plaintext_client_mixed_server', 'tls_client_mixed_server', 'tls_client_tls_server']", diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java new file mode 100644 index 00000000000..4f227ccb6d4 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java @@ -0,0 +1,83 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; + +/** + * A snapshot of the attributes of the file for a given path, and file content if it is a regular file. + * + * @author hakonhall + */ +public class FileSnapshot { + private final Path path; + private final Optional<FileAttributes> attributes; + private final Optional<byte[]> content; + + public static FileSnapshot forPath(Path path) { return forNonExistingFile(path).snapshot(); } + + /** Guaranteed to not throw any exceptions. */ + public static FileSnapshot forNonExistingFile(Path path) { + return new FileSnapshot(path, Optional.empty(), Optional.empty()); + } + + private static FileSnapshot forRegularFile(Path path, FileAttributes attributes, byte[] content) { + if (!attributes.isRegularFile()) throw new IllegalArgumentException(path + " is not a regular file"); + return new FileSnapshot(path, Optional.of(attributes), Optional.of(content)); + } + + private static FileSnapshot forOtherFile(Path path, FileAttributes attributes) { + if (attributes.isRegularFile()) throw new IllegalArgumentException(path + " is a regular file"); + return new FileSnapshot(path, Optional.of(attributes), Optional.empty()); + } + + private FileSnapshot(Path path, Optional<FileAttributes> attributes, Optional<byte[]> content) { + this.path = path; + this.attributes = attributes; + this.content = content; + } + + public Path path() { return path; } + + /** Whether there was a file (or directory) at path. */ + public boolean exists() { return attributes.isPresent(); } + + /** Returns the file attributes if the file exists. */ + public Optional<FileAttributes> attributes() { return attributes; } + + /** Returns the file content if the file exists and is a regular file. */ + public Optional<byte[]> content() { return content; } + + /** Returns the file UTF-8 content if it exists and is a regular file. */ + public Optional<String> utf8Content() { return content.map(c -> new String(c, StandardCharsets.UTF_8)); } + + /** Returns an up-to-date snapshot of the path, possibly {@code this} if last modified time has not changed. */ + public FileSnapshot snapshot() { + Optional<FileAttributes> currentAttributes = new UnixPath(path).getAttributesIfExists(); + if (currentAttributes.isPresent()) { + + // 'this' may still be valid, depending on last modified times. + if (attributes.isPresent()) { + Instant previousModifiedTime = attributes.get().lastModifiedTime(); + Instant currentModifiedTime = currentAttributes.get().lastModifiedTime(); + if (currentModifiedTime.compareTo(previousModifiedTime) <= 0) { + return this; + } + } + + if (currentAttributes.get().isRegularFile()) { + Optional<byte[]> content = IOExceptionUtil.ifExists(() -> Files.readAllBytes(path)); + return content.map(bytes -> FileSnapshot.forRegularFile(path, currentAttributes.get(), bytes)) + // File was removed after getting attributes and before getting content. + .orElseGet(() -> FileSnapshot.forNonExistingFile(path)); + } else { + return FileSnapshot.forOtherFile(path, currentAttributes.get()); + } + } else { + return attributes.isPresent() ? FileSnapshot.forNonExistingFile(path) : this /* avoid allocation */; + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java index f7eba68e455..afc0e7b5c22 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java @@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; import java.util.function.Supplier; /** @@ -17,20 +19,30 @@ public class FileWriter { private final Path path; private final FileSync fileSync; private final PartialFileData.Builder fileDataBuilder = PartialFileData.builder(); - private final Supplier<byte[]> contentProducer; + private final Optional<ByteArraySupplier> contentProducer; private boolean overwriteExistingFile = true; + public FileWriter(Path path) { + this(path, Optional.empty()); + } + public FileWriter(Path path, Supplier<String> contentProducer) { this(path, () -> contentProducer.get().getBytes(StandardCharsets.UTF_8)); } public FileWriter(Path path, ByteArraySupplier contentProducer) { + this(path, Optional.of(contentProducer)); + } + + private FileWriter(Path path, Optional<ByteArraySupplier> contentProducer) { this.path = path; this.fileSync = new FileSync(path); this.contentProducer = contentProducer; } + public Path path() { return path; } + public FileWriter withOwner(String owner) { fileDataBuilder.withOwner(owner); return this; @@ -52,11 +64,19 @@ public class FileWriter { } public boolean converge(TaskContext context) { + return converge(context, contentProducer.orElseThrow().get()); + } + + public boolean converge(TaskContext context, String utf8Content) { + return converge(context, utf8Content.getBytes(StandardCharsets.UTF_8)); + } + + public boolean converge(TaskContext context, byte[] content) { if (!overwriteExistingFile && Files.isRegularFile(path)) { return false; } - fileDataBuilder.withContent(contentProducer.get()); + fileDataBuilder.withContent(content); PartialFileData fileData = fileDataBuilder.create(); return fileSync.convergeTo(context, fileData); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index cf6c6c432f4..2cc74742463 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -231,6 +231,8 @@ public class UnixPath { return new UnixPath(link); } + public FileSnapshot getFileSnapshot() { return FileSnapshot.forPath(path); } + @Override public String toString() { return path.toString(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java new file mode 100644 index 00000000000..8c73d522f1d --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java @@ -0,0 +1,64 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Test; + +import java.nio.file.FileSystem; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author hakonhall + */ +public class FileSnapshotTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final UnixPath path = new UnixPath(fileSystem.getPath("/var/lib/file.txt")); + + private FileSnapshot fileSnapshot = FileSnapshot.forPath(path.toPath()); + + @Test + public void fileDoesNotExist() { + assertFalse(fileSnapshot.exists()); + assertFalse(fileSnapshot.attributes().isPresent()); + assertFalse(fileSnapshot.content().isPresent()); + assertEquals(path.toPath(), fileSnapshot.path()); + } + + @Test + public void directory() { + path.createParents().createDirectory(); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + assertTrue(fileSnapshot.attributes().isPresent()); + assertTrue(fileSnapshot.attributes().get().isDirectory()); + } + + @Test + public void regularFile() { + path.createParents().writeUtf8File("file content"); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + assertTrue(fileSnapshot.attributes().isPresent()); + assertTrue(fileSnapshot.attributes().get().isRegularFile()); + assertTrue(fileSnapshot.utf8Content().isPresent()); + assertEquals("file content", fileSnapshot.utf8Content().get()); + + FileSnapshot newFileSnapshot = fileSnapshot.snapshot(); + assertSame(fileSnapshot, newFileSnapshot); + } + + @Test + public void fileRemoval() { + path.createParents().writeUtf8File("file content"); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + path.deleteIfExists(); + fileSnapshot = fileSnapshot.snapshot(); + assertFalse(fileSnapshot.exists()); + } +}
\ No newline at end of file diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java index 40f12b9c6db..4d7f4b1c397 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -52,6 +52,13 @@ public class SiaUtils { .resolve(String.format("%s.%s.cert.pem", service.getDomainName(), service.getName())); } + public static Path getCaCertificatesFile() { + // The contents of this is the same as /opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem installed + // by the yahoo_certificates_bundle RPM package, except the latter also contains a textual description + // (decoded) of the certificates. + return DEFAULT_SIA_DIRECTORY.resolve("certs").resolve("ca.cert.pem"); + } + public static Optional<PrivateKey> readPrivateKeyFile(AthenzIdentity service) { return readPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); } |