From 505d979eca083b5b937bea246e23cf749f463a68 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 13 Oct 2021 15:08:52 +0200 Subject: Allow setting unix:uid/gid attributes in JimFS --- .../hosted/node/admin/task/util/file/UnixPath.java | 2 +- .../test/file/UnixUidGidAttributeProvider.java | 23 +++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 1121db99399..2ce49ae383a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -149,7 +149,7 @@ public class UnixPath { public UnixPath setGroup(String group) { return setGroup(group, "group"); } public UnixPath setGroupId(int gid) { return setGroup(String.valueOf(gid), "gid"); } - public UnixPath setGroup(String group, String type) { + private UnixPath setGroup(String group, String type) { UserPrincipalLookupService service = path.getFileSystem().getUserPrincipalLookupService(); GroupPrincipal principal = uncheck( () -> service.lookupPrincipalByGroupName(group), diff --git a/testutil/src/main/java/com/yahoo/vespa/test/file/UnixUidGidAttributeProvider.java b/testutil/src/main/java/com/yahoo/vespa/test/file/UnixUidGidAttributeProvider.java index 2b1b0231b4f..13c51851540 100644 --- a/testutil/src/main/java/com/yahoo/vespa/test/file/UnixUidGidAttributeProvider.java +++ b/testutil/src/main/java/com/yahoo/vespa/test/file/UnixUidGidAttributeProvider.java @@ -72,7 +72,8 @@ public class UnixUidGidAttributeProvider extends AttributeProvider { } private int getUniqueId(UserPrincipal user) { - return idCache.computeIfAbsent(user, id -> maybeNumber(id.getName()).orElseGet(uidGenerator::incrementAndGet)); + return maybeNumber(user.getName()) + .orElseGet(() -> idCache.computeIfAbsent(user, id -> uidGenerator.incrementAndGet())); } @SuppressWarnings("unchecked") @@ -106,6 +107,14 @@ public class UnixUidGidAttributeProvider extends AttributeProvider { @Override public void set(File file, String view, String attribute, Object value, boolean create) { + switch (attribute) { + case "uid": + file.setAttribute("owner", "owner", new BasicUserPrincipal(String.valueOf(value))); + return; + case "gid": + file.setAttribute("posix", "group", new BasicGroupPrincipal(String.valueOf(value))); + return; + } throw unsettable(view, attribute); } @@ -158,4 +167,16 @@ public class UnixUidGidAttributeProvider extends AttributeProvider { return Optional.empty(); } } + + private static class BasicUserPrincipal implements UserPrincipal { + private final String name; + private BasicUserPrincipal(String name) { this.name = name; } + + @Override public String getName() { return name; } + @Override public String toString() { return name; } + } + + private static class BasicGroupPrincipal extends BasicUserPrincipal implements GroupPrincipal { + private BasicGroupPrincipal(String name) { super(name); } + } } -- cgit v1.2.3 From 8849307ea12450f40ef00c78e84dbfc65b8f386c Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 13 Oct 2021 15:09:50 +0200 Subject: Create ContainerFileSystem --- .../task/util/fs/ContainerAttributeViews.java | 81 +++++++ .../admin/task/util/fs/ContainerFileSystem.java | 84 +++++++ .../task/util/fs/ContainerFileSystemProvider.java | 265 +++++++++++++++++++++ .../node/admin/task/util/fs/ContainerPath.java | 221 +++++++++++++++++ .../fs/ContainerUserPrincipalLookupService.java | 125 ++++++++++ .../task/util/fs/ContainerFileSystemTest.java | 85 +++++++ .../node/admin/task/util/fs/ContainerPathTest.java | 112 +++++++++ .../ContainerUserPrincipalLookupServiceTest.java | 51 ++++ 8 files changed, 1024 insertions(+) create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java new file mode 100644 index 00000000000..c3246843c75 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java @@ -0,0 +1,81 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import java.io.IOException; +import java.nio.file.ProviderMismatchException; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.UserPrincipal; +import java.util.Map; +import java.util.Set; + +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal; +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal; + +/** + * @author valerijf + */ +class ContainerAttributeViews { + + static class ContainerPosixFileAttributeView implements PosixFileAttributeView { + private final PosixFileAttributeView posixFileAttributeView; + private final ContainerPosixFileAttributes fileAttributes; + + ContainerPosixFileAttributeView(PosixFileAttributeView posixFileAttributeView, + ContainerPosixFileAttributes fileAttributes) { + this.posixFileAttributeView = posixFileAttributeView; + this.fileAttributes = fileAttributes; + } + + @Override public String name() { return "posix"; } + @Override public UserPrincipal getOwner() { return fileAttributes.owner(); } + @Override public PosixFileAttributes readAttributes() { return fileAttributes; } + + @Override + public void setOwner(UserPrincipal owner) throws IOException { + if (!(owner instanceof ContainerUserPrincipal)) throw new ProviderMismatchException(); + posixFileAttributeView.setOwner(((ContainerUserPrincipal) owner).baseFsPrincipal()); + } + + @Override + public void setGroup(GroupPrincipal group) throws IOException { + if (!(group instanceof ContainerGroupPrincipal)) throw new ProviderMismatchException(); + posixFileAttributeView.setGroup(((ContainerGroupPrincipal) group).baseFsPrincipal()); + } + + @Override + public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) throws IOException { + posixFileAttributeView.setTimes(lastModifiedTime, lastAccessTime, createTime); + } + + @Override + public void setPermissions(Set perms) throws IOException { + posixFileAttributeView.setPermissions(perms); + } + } + + static class ContainerPosixFileAttributes implements PosixFileAttributes { + private final Map attributes; + + ContainerPosixFileAttributes(Map attributes) { + this.attributes = attributes; + } + + @SuppressWarnings("unchecked") + @Override public Set permissions() { return (Set) attributes.get("permissions"); } + @Override public ContainerUserPrincipal owner() { return (ContainerUserPrincipal) attributes.get("owner"); } + @Override public ContainerGroupPrincipal group() { return (ContainerGroupPrincipal) attributes.get("group"); } + @Override public FileTime lastModifiedTime() { return (FileTime) attributes.get("lastModifiedTime"); } + @Override public FileTime lastAccessTime() { return (FileTime) attributes.get("lastAccessTime"); } + @Override public FileTime creationTime() { return (FileTime) attributes.get("creationTime"); } + @Override public boolean isRegularFile() { return (boolean) attributes.get("isRegularFile"); } + @Override public boolean isDirectory() { return (boolean) attributes.get("isDirectory"); } + @Override public boolean isSymbolicLink() { return (boolean) attributes.get("isSymbolicLink"); } + @Override public boolean isOther() { return (boolean) attributes.get("isOther"); } + @Override public long size() { return (long) attributes.get("size"); } + @Override public Object fileKey() { return attributes.get("fileKey"); } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java new file mode 100644 index 00000000000..393137f795e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java @@ -0,0 +1,84 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import java.io.IOException; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.WatchService; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.util.Set; + +/** + * @author valerijf + */ +public class ContainerFileSystem extends FileSystem { + + private final ContainerFileSystemProvider containerFsProvider; + + public ContainerFileSystem(ContainerFileSystemProvider containerFsProvider) { + this.containerFsProvider = containerFsProvider; + } + + @Override + public ContainerFileSystemProvider provider() { + return containerFsProvider; + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public boolean isReadOnly() { + return false; + } + + @Override + public String getSeparator() { + return "/"; + } + + @Override + public Set supportedFileAttributeViews() { + return Set.of("basic", "posix", "unix", "owner"); + } + + @Override + public UserPrincipalLookupService getUserPrincipalLookupService() { + return containerFsProvider.userPrincipalLookupService(); + } + + @Override + public ContainerPath getPath(String first, String... more) { + return ContainerPath.fromPathInContainer(this, Path.of(first, more)); + } + + @Override + public void close() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Iterable getRootDirectories() { + throw new UnsupportedOperationException(); + } + + @Override + public Iterable getFileStores() { + throw new UnsupportedOperationException(); + } + + @Override + public PathMatcher getPathMatcher(String syntaxAndPattern) { + throw new UnsupportedOperationException(); + } + + @Override + public WatchService newWatchService() { + throw new UnsupportedOperationException(); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java new file mode 100644 index 00000000000..79214334c55 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java @@ -0,0 +1,265 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import java.io.IOException; +import java.net.URI; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemAlreadyExistsException; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.ProviderMismatchException; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.UserPrincipal; +import java.nio.file.spi.FileSystemProvider; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerAttributeViews.ContainerPosixFileAttributes; +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerAttributeViews.ContainerPosixFileAttributeView; +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal; +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal; +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author valerijf + */ +public class ContainerFileSystemProvider extends FileSystemProvider { + + private final ContainerFileSystem containerFs; + private final ContainerUserPrincipalLookupService userPrincipalLookupService; + private final Path containerRootOnHost; + + + public ContainerFileSystemProvider(Path containerRootOnHost, int uidOffset, int gidOffset) { + this.containerFs = new ContainerFileSystem(this); + this.userPrincipalLookupService = new ContainerUserPrincipalLookupService( + containerRootOnHost.getFileSystem().getUserPrincipalLookupService(), uidOffset, gidOffset); + this.containerRootOnHost = containerRootOnHost; + } + + public Path containerRootOnHost() { + return containerRootOnHost; + } + + public ContainerUserPrincipalLookupService userPrincipalLookupService() { + return userPrincipalLookupService; + } + + @Override + public String getScheme() { + return "file"; + } + + @Override + public FileSystem newFileSystem(URI uri, Map env) { + throw new FileSystemAlreadyExistsException(); + } + + @Override + public ContainerFileSystem getFileSystem(URI uri) { + return containerFs; + } + + @Override + public Path getPath(URI uri) { + throw new UnsupportedOperationException(); + } + + @Override + public SeekableByteChannel newByteChannel(Path path, Set options, FileAttribute... attrs) throws IOException { + Path pathOnHost = pathOnHost(path); + SeekableByteChannel seekableByteChannel = provider(pathOnHost).newByteChannel(pathOnHost, options, attrs); + fixOwnerToContainerRoot(toContainerPath(path)); + return seekableByteChannel; + } + + @Override + public DirectoryStream newDirectoryStream(Path dir, DirectoryStream.Filter filter) throws IOException { + Path pathOnHost = pathOnHost(dir); + return new ContainerDirectoryStream(provider(pathOnHost).newDirectoryStream(pathOnHost, filter)); + } + + @Override + public void createDirectory(Path dir, FileAttribute... attrs) throws IOException { + Path pathOnHost = pathOnHost(dir); + provider(pathOnHost).createDirectory(pathOnHost); + fixOwnerToContainerRoot(toContainerPath(dir)); + } + + @Override + public void delete(Path path) throws IOException { + Path pathOnHost = pathOnHost(path); + provider(pathOnHost).delete(pathOnHost); + } + + @Override + public void copy(Path source, Path target, CopyOption... options) throws IOException { + // Only called when both 'source' and 'target' have 'this' as the FS provider + Path targetPathOnHost = pathOnHost(target); + provider(targetPathOnHost).copy(pathOnHost(source), targetPathOnHost, options); + fixOwnerToContainerRoot(toContainerPath(target)); + } + + @Override + public void move(Path source, Path target, CopyOption... options) throws IOException { + // Only called when both 'source' and 'target' have 'this' as the FS provider + Path targetPathOnHost = pathOnHost(target); + provider(targetPathOnHost).move(pathOnHost(source), targetPathOnHost, options); + fixOwnerToContainerRoot(toContainerPath(target)); + } + + @Override + public boolean isSameFile(Path path, Path path2) throws IOException { + // 'path' FS provider should be 'this' + if (path2 instanceof ContainerPath) + path2 = pathOnHost(path2); + Path pathOnHost = pathOnHost(path); + return provider(pathOnHost).isSameFile(pathOnHost, path2); + } + + @Override + public boolean isHidden(Path path) throws IOException { + Path pathOnHost = pathOnHost(path); + return provider(pathOnHost).isHidden(pathOnHost); + } + + @Override + public FileStore getFileStore(Path path) { + throw new UnsupportedOperationException(); + } + + @Override + public void checkAccess(Path path, AccessMode... modes) throws IOException { + Path pathOnHost = pathOnHost(path); + provider(pathOnHost).checkAccess(pathOnHost, modes); + } + + @Override + @SuppressWarnings("unchecked") + public V getFileAttributeView(Path path, Class type, LinkOption... options) { + if (!type.isAssignableFrom(PosixFileAttributeView.class)) return null; + Path pathOnHost = pathOnHost(path); + FileSystemProvider provider = pathOnHost.getFileSystem().provider(); + if (type == BasicFileAttributeView.class) // Basic view doesnt have owner/group fields, forward to base FS provider + return provider.getFileAttributeView(pathOnHost, type, options); + + PosixFileAttributeView view = provider.getFileAttributeView(pathOnHost, PosixFileAttributeView.class, options); + return (V) new ContainerPosixFileAttributeView(view, + uncheck(() -> new ContainerPosixFileAttributes(readAttributes(path, "unix:*", options)))); + } + + @Override + @SuppressWarnings("unchecked") + public A readAttributes(Path path, Class type, LinkOption... options) throws IOException { + if (!type.isAssignableFrom(PosixFileAttributes.class)) throw new UnsupportedOperationException(); + Path pathOnHost = pathOnHost(path); + if (type == BasicFileAttributes.class) + return pathOnHost.getFileSystem().provider().readAttributes(pathOnHost, type, options); + + // Non-basic requests need to be upgraded to unix:* to get owner,group,uid,gid fields, which are then re-mapped + return (A) new ContainerPosixFileAttributes(readAttributes(path, "unix:*", options)); + } + + @Override + public Map readAttributes(Path path, String attributes, LinkOption... options) throws IOException { + Path pathOnHost = pathOnHost(path); + int index = attributes.indexOf(':'); + if (index < 0 || attributes.startsWith("basic:")) + return provider(pathOnHost).readAttributes(pathOnHost, attributes, options); + + Map attrs = new HashMap<>(provider(pathOnHost).readAttributes(pathOnHost, "unix:*", options)); + int uid = userPrincipalLookupService.hostUidToContainerUid((int) attrs.get("uid")); + int gid = userPrincipalLookupService.hostGidToContainerGid((int) attrs.get("gid")); + attrs.put("uid", uid); + attrs.put("gid", gid); + attrs.put("owner", new ContainerUserPrincipal(uid, (UserPrincipal) attrs.get("owner"))); + attrs.put("group", new ContainerGroupPrincipal(gid, (GroupPrincipal) attrs.get("group"))); + return attrs; + } + + @Override + public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException { + Path pathOnHost = pathOnHost(path); + provider(pathOnHost).setAttribute(pathOnHost, attribute, fixAttributeValue(attribute, value), options); + } + + private Object fixAttributeValue(String attribute, Object value) { + int index = attribute.indexOf(':'); + if (index > 0) { + switch (attribute.substring(index + 1)) { + case "owner": return cast(value, ContainerUserPrincipal.class).baseFsPrincipal(); + case "group": return cast(value, ContainerGroupPrincipal.class).baseFsPrincipal(); + case "uid": return userPrincipalLookupService.containerUidToHostUid(cast(value, Integer.class)); + case "gid": return userPrincipalLookupService.containerGidToHostGid(cast(value, Integer.class)); + } + } // else basic file attribute + return value; + } + + private void fixOwnerToContainerRoot(ContainerPath path) throws IOException { + setAttribute(path, "unix:uid", 0); + setAttribute(path, "unix:gid", 0); + } + + private class ContainerDirectoryStream implements DirectoryStream { + private final DirectoryStream hostDirectoryStream; + + private ContainerDirectoryStream(DirectoryStream hostDirectoryStream) { + this.hostDirectoryStream = hostDirectoryStream; + } + + @Override + public Iterator iterator() { + Iterator hostPathIterator = hostDirectoryStream.iterator(); + return new Iterator<>() { + @Override + public boolean hasNext() { + return hostPathIterator.hasNext(); + } + + @Override + public Path next() { + Path pathOnHost = hostPathIterator.next(); + return ContainerPath.fromPathOnHost(containerFs, pathOnHost); + } + }; + } + + @Override + public void close() throws IOException { + hostDirectoryStream.close(); + } + } + + + static ContainerPath toContainerPath(Path path) { + return cast(path, ContainerPath.class); + } + + private static T cast(Object value, Class type) { + if (type.isInstance(value)) return type.cast(value); + throw new ProviderMismatchException("Expected " + type.getName() + ", was " + value.getClass().getName()); + } + + private static Path pathOnHost(Path path) { + return toContainerPath(path).pathOnHost(); + } + + private static FileSystemProvider provider(Path path) { + return path.getFileSystem().provider(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java new file mode 100644 index 00000000000..01f18b2088b --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java @@ -0,0 +1,221 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerFileSystemProvider.toContainerPath; + +/** + * @author valerijf + */ +public class ContainerPath implements Path { + private final ContainerFileSystem containerFs; + private final Path pathOnHost; + private final String[] parts; + + private ContainerPath(ContainerFileSystem containerFs, Path pathOnHost, String[] parts) { + this.containerFs = Objects.requireNonNull(containerFs); + this.pathOnHost = Objects.requireNonNull(pathOnHost); + this.parts = Objects.requireNonNull(parts); + + if (!pathOnHost.isAbsolute()) + throw new IllegalArgumentException("Path host must be absolute: " + pathOnHost); + Path containerRootOnHost = containerFs.provider().containerRootOnHost(); + if (!pathOnHost.startsWith(containerRootOnHost)) + throw new IllegalArgumentException("Path on host (" + pathOnHost + ") must start with container root on host (" + containerRootOnHost + ")"); + } + + public Path pathOnHost() { + return pathOnHost; + } + + @Override + public FileSystem getFileSystem() { + return containerFs; + } + + @Override + public ContainerPath getRoot() { + return resolve(containerFs, new String[0], Path.of("/")); + } + + @Override + public Path getFileName() { + if (parts.length == 0) return null; + return Path.of(parts[parts.length - 1]); + } + + @Override + public ContainerPath getParent() { + if (parts.length == 0) return null; + return new ContainerPath(containerFs, pathOnHost.getParent(), Arrays.copyOf(parts, parts.length-1)); + } + + @Override + public int getNameCount() { + return parts.length; + } + + @Override + public Path getName(int index) { + return Path.of(parts[index]); + } + + @Override + public Path subpath(int beginIndex, int endIndex) { + if (beginIndex < 0 || beginIndex >= endIndex || endIndex > parts.length) + throw new IllegalArgumentException(); + if (endIndex - beginIndex == 1) return getName(beginIndex); + + String[] rest = new String[endIndex - beginIndex - 1]; + System.arraycopy(parts, beginIndex + 1, rest, 0, rest.length); + return Path.of(parts[beginIndex], rest); + } + + @Override + public ContainerPath resolve(Path other) { + return resolve(containerFs, parts, other); + } + + @Override + public ContainerPath resolveSibling(String other) { + return resolve(Path.of("..", other)); + } + + @Override + public boolean startsWith(Path other) { + if (other.getFileSystem() != containerFs) return false; + String[] otherParts = toContainerPath(other).parts; + if (parts.length < otherParts.length) return false; + + for (int i = 0; i < otherParts.length; i++) { + if ( ! parts[i].equals(otherParts[i])) return false; + } + return true; + } + + @Override + public boolean endsWith(Path other) { + int offset = parts.length - other.getNameCount(); + // If the other path is longer than this, or the other path is absolute and shorter than this + if (offset < 0 || (other.isAbsolute() && offset > 0)) return false; + + for (int i = 0; i < other.getNameCount(); i++) { + if ( ! parts[offset + i].equals(other.getName(i).toString())) return false; + } + return true; + } + + @Override + public boolean isAbsolute() { + // All container paths are normalized and absolute + return true; + } + + @Override + public ContainerPath normalize() { + // All container paths are normalized and absolute + return this; + } + + @Override + public ContainerPath toAbsolutePath() { + // All container paths are normalized and absolute + return this; + } + + @Override + public ContainerPath toRealPath(LinkOption... options) throws IOException { + Path realPathOnHost = pathOnHost.toRealPath(options); + if (realPathOnHost.equals(pathOnHost)) return this; + return fromPathOnHost(containerFs, realPathOnHost); + } + + @Override + public Path relativize(Path other) { + return pathOnHost.relativize(toContainerPath(other).pathOnHost); + } + + @Override + public URI toUri() { + throw new UnsupportedOperationException(); + } + + @Override + public WatchKey register(WatchService watcher, WatchEvent.Kind[] events, WatchEvent.Modifier... modifiers) throws IOException { + return pathOnHost.register(watcher, events, modifiers); + } + + @Override + public int compareTo(Path other) { + return pathOnHost.compareTo(toContainerPath(other)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContainerPath paths = (ContainerPath) o; + return containerFs.equals(paths.containerFs) && pathOnHost.equals(paths.pathOnHost) && Arrays.equals(parts, paths.parts); + } + + @Override + public int hashCode() { + int result = Objects.hash(containerFs, pathOnHost); + result = 31 * result + Arrays.hashCode(parts); + return result; + } + + @Override + public String toString() { + return '/' + String.join("/", parts); + } + + private static ContainerPath resolve(ContainerFileSystem containerFs, String[] currentParts, Path other) { + List parts = other.isAbsolute() ? new ArrayList<>() : new ArrayList<>(Arrays.asList(currentParts)); + for (int i = 0; i < other.getNameCount(); i++) { + String part = other.getName(i).toString(); + if (part.isEmpty() || part.equals(".")) continue; + if (part.equals("..")) { + if (!parts.isEmpty()) parts.remove(parts.size() - 1); + continue; + } + parts.add(part); + } + + return new ContainerPath(containerFs, + containerFs.provider().containerRootOnHost().resolve(String.join("/", parts)), + parts.toArray(String[]::new)); + } + + static ContainerPath fromPathInContainer(ContainerFileSystem containerFs, Path pathInContainer) { + if (!pathInContainer.isAbsolute()) + throw new IllegalArgumentException("Path in container must be absolute: " + pathInContainer); + return resolve(containerFs, new String[0], pathInContainer); + } + + static ContainerPath fromPathOnHost(ContainerFileSystem containerFs, Path pathOnHost) { + pathOnHost = pathOnHost.normalize(); + Path containerRootOnHost = containerFs.provider().containerRootOnHost(); + Path pathUnderContainerStore = containerRootOnHost.relativize(pathOnHost); + List parts = new ArrayList<>(); + for (int i = 0; i < pathUnderContainerStore.getNameCount(); i++) { + String part = pathUnderContainerStore.getName(i).toString(); + if (part.isEmpty() || part.equals(".")) continue; + if (part.equals("..")) throw new IllegalArgumentException("Path " + pathOnHost + " is not under container root " + containerRootOnHost); + parts.add(part); + } + return new ContainerPath(containerFs, pathOnHost, parts.toArray(String[]::new)); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java new file mode 100644 index 00000000000..aefb12d7d4b --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java @@ -0,0 +1,125 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import java.io.IOException; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.UserPrincipal; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.nio.file.attribute.UserPrincipalNotFoundException; +import java.util.Objects; + +/** + * @author valerijf + */ +class ContainerUserPrincipalLookupService extends UserPrincipalLookupService { + + /** Total number of UID/GID that are mapped for each container */ + private static final int ID_RANGE = 1 << 16; + + /** + * IDs outside the ID range are translated to the overflow ID before being written to disk: + * https://github.com/torvalds/linux/blob/5bfc75d92efd494db37f5c4c173d3639d4772966/Documentation/admin-guide/sysctl/fs.rst#overflowgid--overflowuid */ + static final int OVERFLOW_ID = 65_534; + + private final UserPrincipalLookupService baseFsUserPrincipalLookupService; + private final int uidOffset; + private final int gidOffset; + + ContainerUserPrincipalLookupService(UserPrincipalLookupService baseFsUserPrincipalLookupService, int uidOffset, int gidOffset) { + this.baseFsUserPrincipalLookupService = baseFsUserPrincipalLookupService; + this.uidOffset = uidOffset; + this.gidOffset = gidOffset; + } + + public int containerUidToHostUid(int containerUid) { return containerIdToHostId(containerUid, uidOffset); } + public int containerGidToHostGid(int containerGid) { return containerIdToHostId(containerGid, gidOffset); } + public int hostUidToContainerUid(int hostUid) { return hostIdToContainerId(hostUid, uidOffset); } + public int hostGidToContainerGid(int hostGid) { return hostIdToContainerId(hostGid, gidOffset); } + + @Override + public ContainerUserPrincipal lookupPrincipalByName(String name) throws IOException { + int containerUid = resolve(name); + String hostUid = String.valueOf(containerUidToHostUid(containerUid)); + return new ContainerUserPrincipal(containerUid, baseFsUserPrincipalLookupService.lookupPrincipalByName(hostUid)); + } + + @Override + public ContainerGroupPrincipal lookupPrincipalByGroupName(String group) throws IOException { + int containerGid = resolve(group); + String hostGid = String.valueOf(containerGidToHostGid(containerGid)); + return new ContainerGroupPrincipal(containerGid, baseFsUserPrincipalLookupService.lookupPrincipalByGroupName(hostGid)); + } + + private static int resolve(String name) throws UserPrincipalNotFoundException { + if (name.equals("root")) return 0; + if (name.equals("vespa")) return 1000; + + try { + return Integer.parseInt(name); + } catch (NumberFormatException ignored) { + throw new UserPrincipalNotFoundException(name); + } + } + + private abstract static class NamedPrincipal implements UserPrincipal { + private final int id; + private final String name; + private final UserPrincipal baseFsPrincipal; + + private NamedPrincipal(int id, UserPrincipal baseFsPrincipal) { + this.id = id; + this.name = Integer.toString(id); + this.baseFsPrincipal = Objects.requireNonNull(baseFsPrincipal); + } + + @Override + public final String getName() { + return name; + } + + public int id() { + return id; + } + + public UserPrincipal baseFsPrincipal() { + return baseFsPrincipal; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NamedPrincipal that = (NamedPrincipal) o; + return id == that.id && baseFsPrincipal.equals(that.baseFsPrincipal); + } + + @Override + public int hashCode() { + return Objects.hash(id, baseFsPrincipal); + } + } + + static final class ContainerUserPrincipal extends NamedPrincipal { + ContainerUserPrincipal(int id, UserPrincipal baseFsPrincipal) { super(id, baseFsPrincipal); } + + @Override public String toString() { return "{id=" + id() + ", baseFsPrincipal=" + baseFsPrincipal() + '}'; } + } + + static final class ContainerGroupPrincipal extends NamedPrincipal implements GroupPrincipal { + ContainerGroupPrincipal(int id, GroupPrincipal baseFsPrincipal) { super(id, baseFsPrincipal); } + + @Override public GroupPrincipal baseFsPrincipal() { return (GroupPrincipal) super.baseFsPrincipal(); } + @Override public String toString() { return "{" + "id=" + id() + ", baseFsPrincipal=" + baseFsPrincipal() + '}'; } + } + + private static int containerIdToHostId(int id, int idOffset) { + if (id < 0 || id > ID_RANGE) + throw new IllegalArgumentException("Invalid container id: " + id); + return idOffset + id; + } + + private static int hostIdToContainerId(int id, int idOffset) { + id = id - idOffset; + return id < 0 || id >= ID_RANGE ? OVERFLOW_ID : id; + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java new file mode 100644 index 00000000000..e71ba59ce39 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java @@ -0,0 +1,85 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.UserPrincipal; +import java.util.Map; + +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.OVERFLOW_ID; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author valerijf + */ +class ContainerFileSystemTest { + + private final FileSystem fileSystem = TestFileSystem.create(); + private final UnixPath containerRootOnHost = new UnixPath(fileSystem.getPath("/data/storage/ctr1")); + private final ContainerFileSystem containerFs = new ContainerFileSystemProvider(containerRootOnHost.createDirectories().toPath(), 10_000, 11_000).getFileSystem(null); + + @Test + public void creates_files_and_directories_with_container_root_as_owner() throws IOException { + ContainerPath containerPath = ContainerPath.fromPathInContainer(containerFs, Path.of("/opt/vespa/logs/file")); + UnixPath unixPath = new UnixPath(containerPath).createParents().writeUtf8File("hello world"); + + for (ContainerPath p = containerPath; p.getParent() != null; p = p.getParent()) + assertOwnership(p, 0, 0, 10000, 11000); + + unixPath.setOwnerId(500).setGroupId(1000); + assertOwnership(containerPath, 500, 1000, 10500, 12000); + + UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); + ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/copy1")); + Files.copy(hostFile.toPath(), destination); + assertOwnership(destination, 0, 0, 10000, 11000); + } + + @Test + public void copy() throws IOException { + UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); + ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest")); + + // If file is copied to JimFS path, the UID/GIDs are not fixed + Files.copy(hostFile.toPath(), destination.pathOnHost()); + assertEquals(String.valueOf(OVERFLOW_ID), Files.getOwner(destination).getName()); + Files.delete(destination); + + Files.copy(hostFile.toPath(), destination); + assertOwnership(destination, 0, 0, 10000, 11000); + } + + @Test + public void move() throws IOException { + UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); + ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest")); + + // If file is moved to JimFS path, the UID/GIDs are not fixed + Files.move(hostFile.toPath(), destination.pathOnHost()); + assertEquals(String.valueOf(OVERFLOW_ID), Files.getOwner(destination).getName()); + Files.delete(destination); + + hostFile.createNewFile(); + Files.move(hostFile.toPath(), destination); + assertOwnership(destination, 0, 0, 10000, 11000); + } + + private static void assertOwnership(ContainerPath path, int contUid, int contGid, int hostUid, int hostGid) throws IOException { + assertOwnership(path, contUid, contGid); + assertOwnership(path.pathOnHost(), hostUid, hostGid); + } + + private static void assertOwnership(Path path, int uid, int gid) throws IOException { + Map attrs = Files.readAttributes(path, "unix:*"); + assertEquals(uid, attrs.get("uid")); + assertEquals(gid, attrs.get("gid")); + assertEquals(String.valueOf(uid), ((UserPrincipal) attrs.get("owner")).getName()); + assertEquals(String.valueOf(gid), ((UserPrincipal) attrs.get("group")).getName()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java new file mode 100644 index 00000000000..ead4ad7ecde --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java @@ -0,0 +1,112 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; + +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath.fromPathInContainer; +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath.fromPathOnHost; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; + +/** + * @author valerijf + */ +class ContainerPathTest { + + private final FileSystem baseFs = TestFileSystem.create(); + private final ContainerFileSystem containerFs = new ContainerFileSystemProvider(baseFs.getPath("/data/storage/ctr1"), 0, 0).getFileSystem(null); + + @Test + public void create_new_container_path() { + ContainerPath path = fromPathInContainer(containerFs, Path.of("/opt/vespa//logs/./file")); + assertPaths(path, "/data/storage/ctr1/opt/vespa/logs/file", "/opt/vespa/logs/file"); + + path = fromPathOnHost(containerFs, baseFs.getPath("/data/storage/ctr1/opt/vespa/logs/file")); + assertPaths(path, "/data/storage/ctr1/opt/vespa/logs/file", "/opt/vespa/logs/file"); + + path = fromPathOnHost(containerFs, baseFs.getPath("/data/storage/ctr2/..////./ctr1/./opt")); + assertPaths(path, "/data/storage/ctr1/opt", "/opt"); + + assertThrows(() -> fromPathInContainer(containerFs, Path.of("relative/path")), "Path in container must be absolute: relative/path"); + assertThrows(() -> fromPathOnHost(containerFs, baseFs.getPath("relative/path")), "Paths have different roots: /data/storage/ctr1, relative/path"); + assertThrows(() -> fromPathOnHost(containerFs, baseFs.getPath("/data/storage/ctr2")), "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); + assertThrows(() -> fromPathOnHost(containerFs, baseFs.getPath("/data/storage/ctr1/../ctr2")), "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); + } + + @Test + public void container_path_operations() { + ContainerPath path = fromPathInContainer(containerFs, Path.of("/opt/vespa/logs/file")); + ContainerPath parent = path.getParent(); + assertPaths(path.getRoot(), "/data/storage/ctr1", "/"); + assertPaths(parent, "/data/storage/ctr1/opt/vespa/logs", "/opt/vespa/logs"); + assertNull(path.getRoot().getParent()); + + assertEquals(Path.of("file"), path.getFileName()); + assertEquals(Path.of("logs"), path.getName(2)); + assertEquals(4, path.getNameCount()); + assertEquals(Path.of("vespa/logs"), path.subpath(1, 3)); + + assertTrue(path.startsWith(path)); + assertTrue(path.startsWith(parent)); + assertFalse(parent.startsWith(path)); + assertFalse(path.startsWith(Path.of(path.toString()))); + + assertTrue(path.endsWith(Path.of(path.toString()))); + assertTrue(path.endsWith(Path.of("logs/file"))); + assertFalse(path.endsWith(Path.of("/logs/file"))); + } + + @Test + public void resolution() { + ContainerPath path = fromPathInContainer(containerFs, Path.of("/opt/vespa/logs")); + assertPaths(path.resolve(Path.of("/root")), "/data/storage/ctr1/root", "/root"); + assertPaths(path.resolve(Path.of("relative")), "/data/storage/ctr1/opt/vespa/logs/relative", "/opt/vespa/logs/relative"); + assertPaths(path.resolve(Path.of("/../../../dir2/../../../dir2")), "/data/storage/ctr1/dir2", "/dir2"); + assertPaths(path.resolve(Path.of("/some/././///path")), "/data/storage/ctr1/some/path", "/some/path"); + + assertPaths(path.resolve(Path.of("../dir")), "/data/storage/ctr1/opt/vespa/dir", "/opt/vespa/dir"); + assertEquals(path.resolve(Path.of("../dir")), path.resolveSibling("dir")); + } + + @Test + public void resolves_real_paths() throws IOException { + ContainerPath path = fromPathInContainer(containerFs, Path.of("/opt/vespa/logs")); + Files.createDirectories(path.pathOnHost().getParent()); + + Files.createFile(baseFs.getPath("/data/storage/ctr1/opt/vespa/target1")); + Files.createSymbolicLink(path.pathOnHost(), path.pathOnHost().resolveSibling("target1")); + assertPaths(path.toRealPath(LinkOption.NOFOLLOW_LINKS), "/data/storage/ctr1/opt/vespa/logs", "/opt/vespa/logs"); + assertPaths(path.toRealPath(), "/data/storage/ctr1/opt/vespa/target1", "/opt/vespa/target1"); + + Files.delete(path.pathOnHost()); + Files.createFile(baseFs.getPath("/data/storage/ctr1/opt/target2")); + Files.createSymbolicLink(path.pathOnHost(), baseFs.getPath("../target2")); + assertPaths(path.toRealPath(), "/data/storage/ctr1/opt/target2", "/opt/target2"); + + Files.delete(path.pathOnHost()); + Files.createFile(baseFs.getPath("/data/storage/ctr2")); + Files.createSymbolicLink(path.pathOnHost(), path.getRoot().pathOnHost().resolveSibling("ctr2")); + assertThrows(path::toRealPath, "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); + } + + private static void assertPaths(ContainerPath actual, String expectedPathOnHost, String expectedPathInContainer) { + assertEquals(expectedPathOnHost, actual.pathOnHost().toString()); + assertEquals(expectedPathInContainer, actual.toString()); + } + + private static void assertThrows(Executable executable, String expectedMsg) { + String actualMsg = Assertions.assertThrows(IllegalArgumentException.class, executable).getMessage(); + assertEquals(expectedMsg, actualMsg); + } +} \ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java new file mode 100644 index 00000000000..27e0beceb37 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.attribute.UserPrincipalNotFoundException; + +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal; +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author valerijf + */ +class ContainerUserPrincipalLookupServiceTest { + + private final ContainerUserPrincipalLookupService userPrincipalLookupService = + new ContainerUserPrincipalLookupService(TestFileSystem.create().getUserPrincipalLookupService(), 1000, 2000); + + @Test + public void correctly_resolves_ids() throws IOException { + ContainerUserPrincipal user = userPrincipalLookupService.lookupPrincipalByName("1000"); + assertEquals("1000", user.getName()); + assertEquals("2000", user.baseFsPrincipal().getName()); + assertEquals(user, userPrincipalLookupService.lookupPrincipalByName("vespa")); + + ContainerGroupPrincipal group = userPrincipalLookupService.lookupPrincipalByGroupName("1000"); + assertEquals("1000", group.getName()); + assertEquals("3000", group.baseFsPrincipal().getName()); + assertEquals(group, userPrincipalLookupService.lookupPrincipalByGroupName("vespa")); + + assertThrows(UserPrincipalNotFoundException.class, () -> userPrincipalLookupService.lookupPrincipalByName("test")); + } + + @Test + public void translates_between_ids() { + assertEquals(1001, userPrincipalLookupService.containerUidToHostUid(1)); + assertEquals(2001, userPrincipalLookupService.containerGidToHostGid(1)); + assertEquals(1, userPrincipalLookupService.hostUidToContainerUid(1001)); + assertEquals(1, userPrincipalLookupService.hostGidToContainerGid(2001)); + + assertEquals(65_534, userPrincipalLookupService.hostUidToContainerUid(1)); + assertEquals(65_534, userPrincipalLookupService.hostUidToContainerUid(999999)); + + assertThrows(IllegalArgumentException.class, () -> userPrincipalLookupService.containerUidToHostUid(-1)); + assertThrows(IllegalArgumentException.class, () -> userPrincipalLookupService.containerUidToHostUid(70_000)); + } +} \ No newline at end of file -- cgit v1.2.3 From 4fb42f0a38fcbd411c4e1074a3e2a8ddf595081f Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 13 Oct 2021 16:32:00 +0200 Subject: Move identical toString() implementation to base class --- .../admin/task/util/fs/ContainerUserPrincipalLookupService.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java index aefb12d7d4b..d52d9c75661 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java @@ -97,19 +97,21 @@ class ContainerUserPrincipalLookupService extends UserPrincipalLookupService { public int hashCode() { return Objects.hash(id, baseFsPrincipal); } + + @Override + public String toString() { + return "{id=" + id + ", baseFsPrincipal=" + baseFsPrincipal + '}'; + } } static final class ContainerUserPrincipal extends NamedPrincipal { ContainerUserPrincipal(int id, UserPrincipal baseFsPrincipal) { super(id, baseFsPrincipal); } - - @Override public String toString() { return "{id=" + id() + ", baseFsPrincipal=" + baseFsPrincipal() + '}'; } } static final class ContainerGroupPrincipal extends NamedPrincipal implements GroupPrincipal { ContainerGroupPrincipal(int id, GroupPrincipal baseFsPrincipal) { super(id, baseFsPrincipal); } @Override public GroupPrincipal baseFsPrincipal() { return (GroupPrincipal) super.baseFsPrincipal(); } - @Override public String toString() { return "{" + "id=" + id() + ", baseFsPrincipal=" + baseFsPrincipal() + '}'; } } private static int containerIdToHostId(int id, int idOffset) { -- cgit v1.2.3 From d1028019641d396f35fcc81115bd7452e757ad7a Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 13 Oct 2021 17:14:48 +0200 Subject: Simplify fromPathOnHost --- .../node/admin/task/util/fs/ContainerPath.java | 28 ++++++++++++---------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java index 01f18b2088b..e967806dc55 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java @@ -205,17 +205,19 @@ public class ContainerPath implements Path { return resolve(containerFs, new String[0], pathInContainer); } - static ContainerPath fromPathOnHost(ContainerFileSystem containerFs, Path pathOnHost) { - pathOnHost = pathOnHost.normalize(); - Path containerRootOnHost = containerFs.provider().containerRootOnHost(); - Path pathUnderContainerStore = containerRootOnHost.relativize(pathOnHost); - List parts = new ArrayList<>(); - for (int i = 0; i < pathUnderContainerStore.getNameCount(); i++) { - String part = pathUnderContainerStore.getName(i).toString(); - if (part.isEmpty() || part.equals(".")) continue; - if (part.equals("..")) throw new IllegalArgumentException("Path " + pathOnHost + " is not under container root " + containerRootOnHost); - parts.add(part); - } - return new ContainerPath(containerFs, pathOnHost, parts.toArray(String[]::new)); - } +static ContainerPath fromPathOnHost(ContainerFileSystem containerFs, Path pathOnHost) { + pathOnHost = pathOnHost.normalize(); + Path containerRootOnHost = containerFs.provider().containerRootOnHost(); + Path pathUnderContainerStorage = containerRootOnHost.relativize(pathOnHost); + + if (pathUnderContainerStorage.getNameCount() == 0 || pathUnderContainerStorage.getName(0).toString().isEmpty()) + return new ContainerPath(containerFs, pathOnHost, new String[0]); + if (pathUnderContainerStorage.getName(0).toString().equals("..")) + throw new IllegalArgumentException("Path " + pathOnHost + " is not under container root " + containerRootOnHost); + + List parts = new ArrayList<>(); + for (int i = 0; i < pathUnderContainerStorage.getNameCount(); i++) + parts.add(pathUnderContainerStorage.getName(i).toString()); + return new ContainerPath(containerFs, pathOnHost, parts.toArray(String[]::new)); +} } -- cgit v1.2.3 From 8af3bc3d5dd5a81683cb5c5f82d1738190203801 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 13 Oct 2021 17:15:14 +0200 Subject: Set owner and group for known IDs inside container NS --- .../task/util/fs/ContainerUserPrincipalLookupService.java | 14 +++++++++++--- .../node/admin/task/util/fs/ContainerFileSystemTest.java | 3 --- .../util/fs/ContainerUserPrincipalLookupServiceTest.java | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java index d52d9c75661..893e86ca239 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java @@ -1,12 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.fs; +import com.google.common.collect.ImmutableBiMap; + import java.io.IOException; import java.nio.file.attribute.GroupPrincipal; import java.nio.file.attribute.UserPrincipal; import java.nio.file.attribute.UserPrincipalLookupService; import java.nio.file.attribute.UserPrincipalNotFoundException; import java.util.Objects; +import java.util.Optional; /** * @author valerijf @@ -21,6 +24,11 @@ class ContainerUserPrincipalLookupService extends UserPrincipalLookupService { * https://github.com/torvalds/linux/blob/5bfc75d92efd494db37f5c4c173d3639d4772966/Documentation/admin-guide/sysctl/fs.rst#overflowgid--overflowuid */ static final int OVERFLOW_ID = 65_534; + private static final ImmutableBiMap CONTAINER_IDS_BY_NAME = ImmutableBiMap.builder() + .put("root", 0) + .put("vespa", 1000) + .build(); + private final UserPrincipalLookupService baseFsUserPrincipalLookupService; private final int uidOffset; private final int gidOffset; @@ -51,8 +59,8 @@ class ContainerUserPrincipalLookupService extends UserPrincipalLookupService { } private static int resolve(String name) throws UserPrincipalNotFoundException { - if (name.equals("root")) return 0; - if (name.equals("vespa")) return 1000; + Integer id = CONTAINER_IDS_BY_NAME.get(name); + if (id != null) return id; try { return Integer.parseInt(name); @@ -68,7 +76,7 @@ class ContainerUserPrincipalLookupService extends UserPrincipalLookupService { private NamedPrincipal(int id, UserPrincipal baseFsPrincipal) { this.id = id; - this.name = Integer.toString(id); + this.name = Optional.ofNullable(CONTAINER_IDS_BY_NAME.inverse().get(id)).orElseGet(() -> Integer.toString(id)); this.baseFsPrincipal = Objects.requireNonNull(baseFsPrincipal); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java index e71ba59ce39..38c1e2720c3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java @@ -9,7 +9,6 @@ import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.UserPrincipal; import java.util.Map; import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.OVERFLOW_ID; @@ -79,7 +78,5 @@ class ContainerFileSystemTest { Map attrs = Files.readAttributes(path, "unix:*"); assertEquals(uid, attrs.get("uid")); assertEquals(gid, attrs.get("gid")); - assertEquals(String.valueOf(uid), ((UserPrincipal) attrs.get("owner")).getName()); - assertEquals(String.valueOf(gid), ((UserPrincipal) attrs.get("group")).getName()); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java index 27e0beceb37..a459c24049e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java @@ -23,12 +23,12 @@ class ContainerUserPrincipalLookupServiceTest { @Test public void correctly_resolves_ids() throws IOException { ContainerUserPrincipal user = userPrincipalLookupService.lookupPrincipalByName("1000"); - assertEquals("1000", user.getName()); + assertEquals("vespa", user.getName()); assertEquals("2000", user.baseFsPrincipal().getName()); assertEquals(user, userPrincipalLookupService.lookupPrincipalByName("vespa")); ContainerGroupPrincipal group = userPrincipalLookupService.lookupPrincipalByGroupName("1000"); - assertEquals("1000", group.getName()); + assertEquals("vespa", group.getName()); assertEquals("3000", group.baseFsPrincipal().getName()); assertEquals(group, userPrincipalLookupService.lookupPrincipalByGroupName("vespa")); -- cgit v1.2.3