Merge pull request #19651 from vespa-engine/freva/reapply-cache-fs

Reapply "Do not re-create Container FS for each NodeAgentContext"

17 files changed, 145 insertions, 96 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
index 280e58c91f1..6fef2cbbb50 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
@@ -20,10 +20,11 @@ import com.yahoo.vespa.athenz.identityprovider.client.CsrGenerator;
 import com.yahoo.vespa.athenz.identityprovider.client.DefaultIdentityDocumentClient;
 import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
 import com.yahoo.vespa.athenz.utils.SiaUtils;
-import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentTask;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.VespaUser;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
 import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
@@ -206,7 +207,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
                             EntityBindingsMapper.toAttestationData(signedIdentityDocument),
                             csr);
             EntityBindingsMapper.writeSignedIdentityDocumentToFile(identityDocumentFile, signedIdentityDocument);
-            writePrivateKeyAndCertificate(context.userNamespace().vespaUserId(),
+            writePrivateKeyAndCertificate(context.vespaUser(),
                     privateKeyFile, keyPair.getPrivate(), certificateFile, instanceIdentity.certificate());
             context.log(logger, "Instance successfully registered and credentials written to file");
         }
@@ -234,7 +235,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
                                 context.identity(),
                                 identityDocument.providerUniqueId().asDottedString(),
                                 csr);
-                writePrivateKeyAndCertificate(context.userNamespace().vespaUserId(),
+                writePrivateKeyAndCertificate(context.vespaUser(),
                         privateKeyFile, keyPair.getPrivate(), certificateFile, instanceIdentity.certificate());
                 context.log(logger, "Instance successfully refreshed and credentials written to file");
             } catch (ZtsClientException e) {
@@ -251,19 +252,19 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
     }
 
 
-    private static void writePrivateKeyAndCertificate(int vespaUid,
+    private static void writePrivateKeyAndCertificate(VespaUser vespaUser,
                                                       ContainerPath privateKeyFile,
                                                       PrivateKey privateKey,
                                                       ContainerPath certificateFile,
                                                       X509Certificate certificate) {
-        writeFile(privateKeyFile, vespaUid, KeyUtils.toPem(privateKey));
-        writeFile(certificateFile, vespaUid, X509CertificateUtils.toPem(certificate));
+        writeFile(privateKeyFile, vespaUser, KeyUtils.toPem(privateKey));
+        writeFile(certificateFile, vespaUser, X509CertificateUtils.toPem(certificate));
     }
 
-    private static void writeFile(ContainerPath path, int vespaUid, String utf8Content) {
+    private static void writeFile(ContainerPath path, VespaUser vespaUser, String utf8Content) {
         new UnixPath(path.resolveSibling(path.getFileName() + ".tmp"))
                 .writeUtf8File(utf8Content, "r--------")
-                .setOwnerId(vespaUid)
+                .setOwnerId(vespaUser.uid())
                 .atomicMove(path);
     }
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java
index 0a9496be0a6..b299e1f3f0d 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java
@@ -101,8 +101,8 @@ public class VespaServiceDumperImpl implements VespaServiceDumper {
             }
             context.log(log, Level.INFO, "Creating '" + unixPathDirectory +"'.");
             unixPathDirectory.createDirectory("rwxr-x---")
-                    .setOwner(context.userNamespace().vespaUser())
-                    .setGroup(context.userNamespace().vespaGroup());
+                    .setOwner(context.vespaUser().name())
+                    .setGroup(context.vespaUser().group());
             URI destination = serviceDumpDestination(nodeSpec, createDumpId(request));
             ProducerContext producerCtx = new ProducerContext(context, directory, request);
             List<Artifact> producedArtifacts = new ArrayList<>();
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
index 53c9e741f59..dda404797d9 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin;
 
 import com.yahoo.concurrent.ThreadFactoryFactory;
 import com.yahoo.config.provision.HostName;
-import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
@@ -13,7 +12,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory;
 import com.yahoo.yolean.Exceptions;
 
-import java.time.Clock;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.EnumSet;
@@ -59,9 +57,7 @@ public class NodeAdminStateUpdater {
             NodeRepository nodeRepository,
             Orchestrator orchestrator,
             NodeAdmin nodeAdmin,
-            HostName hostHostname,
-            Clock clock,
-            FlagSource flagSource) {
+            HostName hostHostname) {
         this.nodeAgentContextFactory = nodeAgentContextFactory;
         this.nodeRepository = nodeRepository;
         this.orchestrator = orchestrator;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
index 8cf8553bc34..23a81458134 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
@@ -6,10 +6,10 @@ import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.container.ContainerNetworkMode;
 import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
 
@@ -42,7 +42,8 @@ public interface NodeAgentContext extends TaskContext {
 
     ZoneApi zone();
 
-    UserNamespace userNamespace();
+    /** @return information about the Vespa user inside the container */
+    VespaUser vespaUser();
 
     default boolean isDisabled(NodeAgentTask task) {
         return false;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
index 9bcf5d58d6e..0b1f7f24ced 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
@@ -12,9 +12,9 @@ import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.flags.PermanentFlags;
-import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.container.ContainerNetworkMode;
 import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerFileSystem;
 import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
@@ -42,16 +42,14 @@ public class NodeAgentContextImpl implements NodeAgentContext {
     private final ZoneApi zone;
     private final ContainerFileSystem containerFs;
     private final ContainerPath pathToVespaHome;
-    private final UserNamespace userNamespace;
     private final double cpuSpeedup;
     private final Set<NodeAgentTask> disabledNodeAgentTasks;
     private final Optional<ApplicationId> hostExclusiveTo;
 
     public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzIdentity identity,
                                 ContainerNetworkMode containerNetworkMode, ZoneApi zone,
-                                FlagSource flagSource, Path pathToContainerStorage, String pathToVespaHome,
-                                UserNamespace userNamespace, double cpuSpeedup,
-                                Optional<ApplicationId> hostExclusiveTo) {
+                                FlagSource flagSource, ContainerFileSystem containerFs, String pathToVespaHome,
+                                double cpuSpeedup, Optional<ApplicationId> hostExclusiveTo) {
         if (cpuSpeedup <= 0)
             throw new IllegalArgumentException("cpuSpeedUp must be positive, was: " + cpuSpeedup);
 
@@ -61,10 +59,9 @@ public class NodeAgentContextImpl implements NodeAgentContext {
         this.identity = Objects.requireNonNull(identity);
         this.containerNetworkMode = Objects.requireNonNull(containerNetworkMode);
         this.zone = Objects.requireNonNull(zone);
-        this.containerFs = ContainerFileSystem.create(pathToContainerStorage.resolve(containerName.asString()), userNamespace);
+        this.containerFs = Objects.requireNonNull(containerFs);
         this.pathToVespaHome = containerFs.getPath(pathToVespaHome);
         this.logPrefix = containerName.asString() + ": ";
-        this.userNamespace = Objects.requireNonNull(userNamespace);
         this.cpuSpeedup = cpuSpeedup;
         this.disabledNodeAgentTasks = NodeAgentTask.fromString(
                 PermanentFlags.DISABLED_HOST_ADMIN_TASKS.bindTo(flagSource).with(FetchVector.Dimension.HOSTNAME, node.hostname()).value());
@@ -102,8 +99,8 @@ public class NodeAgentContextImpl implements NodeAgentContext {
     }
 
     @Override
-    public UserNamespace userNamespace() {
-        return userNamespace;
+    public VespaUser vespaUser() {
+        return containerFs.getUserPrincipalLookupService().vespaUser();
     }
 
     @Override
@@ -191,6 +188,7 @@ public class NodeAgentContextImpl implements NodeAgentContext {
         private ContainerNetworkMode containerNetworkMode;
         private ZoneApi zone;
         private UserNamespace userNamespace;
+        private VespaUser vespaUser;
         private Path containerStorage;
         private FlagSource flagSource;
         private double cpuSpeedUp = 1;
@@ -230,6 +228,12 @@ public class NodeAgentContextImpl implements NodeAgentContext {
             return this;
         }
 
+        public Builder vespaUser(VespaUser vespaUser) {
+            this.vespaUser = vespaUser;
+            return this;
+        }
+
+
         /** Sets the file system to use for paths. */
         public Builder fileSystem(FileSystem fileSystem) {
             return containerStorage(fileSystem.getPath(DEFAULT_CONTAINER_STORAGE.toString()));
@@ -258,6 +262,14 @@ public class NodeAgentContextImpl implements NodeAgentContext {
         public NodeAgentContextImpl build() {
             Objects.requireNonNull(containerStorage, "Must set one of containerStorage or fileSystem");
 
+            UserNamespace userNamespace = Optional.ofNullable(this.userNamespace)
+                    .orElseGet(() -> new UserNamespace(100000, 100000));
+            VespaUser vespaUser = Optional.ofNullable(this.vespaUser)
+                    .orElseGet(() -> new VespaUser("vespa", "vespa", 1000, 100));
+            ContainerFileSystem containerFs = ContainerFileSystem.create(containerStorage
+                    .resolve(nodeSpecBuilder.hostname().split("\\.")[0]), userNamespace, vespaUser);
+            containerFs.createRoot();
+
             return new NodeAgentContextImpl(
                     nodeSpecBuilder.build(),
                     Optional.ofNullable(acl).orElse(Acl.EMPTY),
@@ -285,9 +297,8 @@ public class NodeAgentContextImpl implements NodeAgentContext {
                         }
                     }),
                     Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new),
-                    containerStorage,
+                    containerFs,
                     "/opt/vespa",
-                    Optional.ofNullable(userNamespace).orElseGet(() -> new UserNamespace(100000, 100000, "vespa", "vespa", 1000, 100)),
                     cpuSpeedUp, hostExclusiveTo);
         }
     }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java
index 1a25b5c3c5e..005452411bd 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java
@@ -1,8 +1,6 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.nodeagent;
 
-import java.util.Objects;
-
 /**
  * @author valerijf
  */
@@ -16,20 +14,12 @@ public class UserNamespace {
      * https://github.com/torvalds/linux/blob/5bfc75d92efd494db37f5c4c173d3639d4772966/Documentation/admin-guide/sysctl/fs.rst#overflowgid--overflowuid */
     private static final int OVERFLOW_ID = 65_534;
 
-    private final int uidOffset;
-    private final int gidOffset;
-    private final String vespaUser;
-    private final String vespaGroup;
-    private final int vespaUserId;
-    private final int vespaGroupId;
+    private volatile int uidOffset;
+    private volatile int gidOffset;
 
-    public UserNamespace(int uidOffset, int gidOffset, String vespaUser, String vespaGroup, int vespaUserId, int vespaGroupId) {
+    public UserNamespace(int uidOffset, int gidOffset) {
         this.uidOffset = uidOffset;
         this.gidOffset = gidOffset;
-        this.vespaUser = Objects.requireNonNull(vespaUser);
-        this.vespaGroup = Objects.requireNonNull(vespaGroup);
-        this.vespaUserId = vespaUserId;
-        this.vespaGroupId = vespaGroupId;
     }
 
     public int userIdOnHost(int containerUid) { return toHostId(containerUid, uidOffset); }
@@ -37,14 +27,15 @@ public class UserNamespace {
     public int userIdInContainer(int hostUid) { return toContainerId(hostUid, uidOffset); }
     public int groupIdInContainer(int hostGid) { return toContainerId(hostGid, gidOffset); }
 
-    public String vespaUser() { return vespaUser; }
-    public String vespaGroup() { return vespaGroup; }
-    public int vespaUserId() { return vespaUserId; }
-    public int vespaGroupId() { return vespaGroupId; }
-
     public int idRange() { return ID_RANGE; }
     public int overflowId() { return OVERFLOW_ID; }
 
+    // Remove after migration to mapped namespaces is complete, make fields final
+    public void setOffsets(int idOffset) {
+        this.uidOffset = idOffset;
+        this.gidOffset = idOffset;
+    }
+
     private static int toHostId(int containerId, int idOffset) {
         if (containerId < 0 || containerId > ID_RANGE)
             throw new IllegalArgumentException("Invalid container id: " + containerId);
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/VespaUser.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/VespaUser.java
new file mode 100644
index 00000000000..78ccca80beb
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/VespaUser.java
@@ -0,0 +1,29 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import java.util.Objects;
+
+/**
+ * Describes Vespa user inside the container user namespace.
+ * 
+ * @author valerijf
+ */
+public class VespaUser {
+
+    private final String name;
+    private final String group;
+    private final int uid;
+    private final int gid;
+
+    public VespaUser(String name, String group, int uid, int gid) {
+        this.name = Objects.requireNonNull(name);
+        this.group = Objects.requireNonNull(group);
+        this.uid = uid;
+        this.gid = gid;
+    }
+
+    public String name() { return name; }
+    public String group() { return group; }
+    public int uid() { return uid; }
+    public int gid() { return gid; }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java
index 078a60ba7a5..6f0ab5576b7 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java
@@ -2,28 +2,35 @@
 package com.yahoo.vespa.hosted.node.admin.task.util.fs;
 
 import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.VespaUser;
 
 import java.io.IOException;
 import java.nio.file.FileStore;
 import java.nio.file.FileSystem;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.PathMatcher;
 import java.nio.file.WatchService;
-import java.nio.file.attribute.UserPrincipalLookupService;
 import java.util.Set;
 
-import static com.yahoo.yolean.Exceptions.uncheck;
-
 /**
  * @author valerijf
  */
 public class ContainerFileSystem extends FileSystem {
 
     private final ContainerFileSystemProvider containerFsProvider;
+    private final Path containerRootOnHost;
 
-    ContainerFileSystem(ContainerFileSystemProvider containerFsProvider) {
+    ContainerFileSystem(ContainerFileSystemProvider containerFsProvider, Path containerRootOnHost) {
         this.containerFsProvider = containerFsProvider;
+        this.containerRootOnHost = containerRootOnHost;
+    }
+
+    public Path containerRootOnHost() {
+        return containerRootOnHost;
+    }
+
+    public void createRoot() {
+        provider().createFileSystemRoot();
     }
 
     @Override
@@ -52,7 +59,7 @@ public class ContainerFileSystem extends FileSystem {
     }
 
     @Override
-    public UserPrincipalLookupService getUserPrincipalLookupService() {
+    public ContainerUserPrincipalLookupService getUserPrincipalLookupService() {
         return containerFsProvider.userPrincipalLookupService();
     }
 
@@ -86,8 +93,7 @@ public class ContainerFileSystem extends FileSystem {
         throw new UnsupportedOperationException();
     }
 
-    public static ContainerFileSystem create(Path containerStorageRoot, UserNamespace userNamespace) {
-        uncheck(() -> Files.createDirectories(containerStorageRoot));
-        return new ContainerFileSystemProvider(containerStorageRoot, userNamespace).getFileSystem(null);
+    public static ContainerFileSystem create(Path containerStorageRoot, UserNamespace userNamespace, VespaUser vespaUser) {
+        return new ContainerFileSystemProvider(containerStorageRoot, userNamespace, vespaUser).getFileSystem(null);
     }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java
index 909c6c9cbc1..00dda1d7cd2 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.node.admin.task.util.fs;
 
 import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.VespaUser;
 
 import java.io.IOException;
 import java.net.URI;
@@ -31,8 +32,8 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 
-import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerAttributeViews.ContainerPosixFileAttributes;
 import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerAttributeViews.ContainerPosixFileAttributeView;
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerAttributeViews.ContainerPosixFileAttributes;
 import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal;
 import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal;
 import static com.yahoo.yolean.Exceptions.uncheck;
@@ -43,17 +44,11 @@ import static com.yahoo.yolean.Exceptions.uncheck;
 class ContainerFileSystemProvider extends FileSystemProvider {
     private final ContainerFileSystem containerFs;
     private final ContainerUserPrincipalLookupService userPrincipalLookupService;
-    private final Path containerRootOnHost;
 
-    ContainerFileSystemProvider(Path containerRootOnHost, UserNamespace userNamespace) {
-        this.containerFs = new ContainerFileSystem(this);
+    ContainerFileSystemProvider(Path containerRootOnHost, UserNamespace userNamespace, VespaUser vespaUser) {
+        this.containerFs = new ContainerFileSystem(this, containerRootOnHost);
         this.userPrincipalLookupService = new ContainerUserPrincipalLookupService(
-                containerRootOnHost.getFileSystem().getUserPrincipalLookupService(), userNamespace);
-        this.containerRootOnHost = containerRootOnHost;
-    }
-
-    public Path containerRootOnHost() {
-        return containerRootOnHost;
+                containerRootOnHost.getFileSystem().getUserPrincipalLookupService(), userNamespace, vespaUser);
     }
 
     public ContainerUserPrincipalLookupService userPrincipalLookupService() {
@@ -225,6 +220,16 @@ class ContainerFileSystemProvider extends FileSystemProvider {
         return value;
     }
 
+    void createFileSystemRoot() {
+        ContainerPath root = containerFs.getPath("/");
+        if (!Files.exists(root)) {
+            uncheck(() -> {
+                Files.createDirectories(root.pathOnHost());
+                fixOwnerToContainerRoot(root);
+            });
+        }
+    }
+
     private void fixOwnerToContainerRoot(ContainerPath path) throws IOException {
         setAttribute(path, "unix:uid", 0);
         setAttribute(path, "unix:gid", 0);
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java
index 15295ffd087..853646d53b5 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java
@@ -32,7 +32,7 @@ public class ContainerPath implements Path {
 
         if (!pathOnHost.isAbsolute())
             throw new IllegalArgumentException("Path host must be absolute: " + pathOnHost);
-        Path containerRootOnHost = containerFs.provider().containerRootOnHost();
+        Path containerRootOnHost = containerFs.containerRootOnHost();
         if (!pathOnHost.startsWith(containerRootOnHost))
             throw new IllegalArgumentException("Path on host (" + pathOnHost + ") must start with container root on host (" + containerRootOnHost + ")");
     }
@@ -173,7 +173,7 @@ public class ContainerPath implements Path {
 
     @Override
     public String toString() {
-        return containerFs.provider().containerRootOnHost().getFileName() + ":" + pathInContainer();
+        return containerFs.containerRootOnHost().getFileName() + ":" + pathInContainer();
     }
 
     private static ContainerPath resolve(ContainerFileSystem containerFs, String[] currentParts, Path other) {
@@ -189,7 +189,7 @@ public class ContainerPath implements Path {
         }
 
         return new ContainerPath(containerFs,
-                containerFs.provider().containerRootOnHost().resolve(String.join("/", parts)),
+                containerFs.containerRootOnHost().resolve(String.join("/", parts)),
                 parts.toArray(String[]::new));
     }
 
@@ -201,7 +201,7 @@ public class ContainerPath implements Path {
 
     public static ContainerPath fromPathOnHost(ContainerFileSystem containerFs, Path pathOnHost) {
         pathOnHost = pathOnHost.normalize();
-        Path containerRootOnHost = containerFs.provider().containerRootOnHost();
+        Path containerRootOnHost = containerFs.containerRootOnHost();
         Path pathUnderContainerStorage = containerRootOnHost.relativize(pathOnHost);
 
         if (pathUnderContainerStorage.getNameCount() == 0 || pathUnderContainerStorage.getName(0).toString().isEmpty())
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java
index ae65f6a7f7f..8e35bdccc23 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.node.admin.task.util.fs;
 
 import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.VespaUser;
 
 import java.io.IOException;
 import java.nio.file.attribute.GroupPrincipal;
@@ -13,16 +14,22 @@ import java.util.Objects;
 /**
  * @author valerijf
  */
-class ContainerUserPrincipalLookupService extends UserPrincipalLookupService {
+public class ContainerUserPrincipalLookupService extends UserPrincipalLookupService {
 
     private final UserPrincipalLookupService baseFsUserPrincipalLookupService;
     private final UserNamespace userNamespace;
+    private final VespaUser vespaUser;
 
-    ContainerUserPrincipalLookupService(UserPrincipalLookupService baseFsUserPrincipalLookupService, UserNamespace userNamespace) {
+    ContainerUserPrincipalLookupService(
+            UserPrincipalLookupService baseFsUserPrincipalLookupService, UserNamespace userNamespace, VespaUser vespaUser) {
         this.baseFsUserPrincipalLookupService = Objects.requireNonNull(baseFsUserPrincipalLookupService);
         this.userNamespace = Objects.requireNonNull(userNamespace);
+        this.vespaUser = Objects.requireNonNull(vespaUser);
     }
 
+    public UserNamespace userNamespace() { return userNamespace; }
+    public VespaUser vespaUser() { return vespaUser; }
+
     public int userIdOnHost(int containerUid)  { return userNamespace.userIdOnHost(containerUid); }
     public int groupIdOnHost(int containerGid) { return userNamespace.groupIdOnHost(containerGid); }
     public int userIdInContainer(int hostUid)  { return userNamespace.userIdInContainer(hostUid); }
@@ -30,27 +37,27 @@ class ContainerUserPrincipalLookupService extends UserPrincipalLookupService {
 
     @Override
     public ContainerUserPrincipal lookupPrincipalByName(String name) throws IOException {
-        int containerUid = resolveName(name, userNamespace.vespaUser(), userNamespace.vespaUserId());
-        String user = resolveId(containerUid, userNamespace.vespaUser(), userNamespace.vespaUserId());
+        int containerUid = resolveName(name, vespaUser.name(), vespaUser.uid());
+        String user = resolveId(containerUid, vespaUser.name(), vespaUser.uid());
         String hostUid = String.valueOf(userIdOnHost(containerUid));
         return new ContainerUserPrincipal(containerUid, user, baseFsUserPrincipalLookupService.lookupPrincipalByName(hostUid));
     }
 
     @Override
     public ContainerGroupPrincipal lookupPrincipalByGroupName(String group) throws IOException {
-        int containerGid = resolveName(group, userNamespace.vespaGroup(), userNamespace.vespaGroupId());
-        String name = resolveId(containerGid, userNamespace.vespaGroup(), userNamespace.vespaGroupId());
+        int containerGid = resolveName(group, vespaUser.group(), vespaUser.gid());
+        String name = resolveId(containerGid, vespaUser.group(), vespaUser.gid());
         String hostGid = String.valueOf(groupIdOnHost(containerGid));
         return new ContainerGroupPrincipal(containerGid, name, baseFsUserPrincipalLookupService.lookupPrincipalByGroupName(hostGid));
     }
 
     public ContainerUserPrincipal userPrincipal(int uid, UserPrincipal baseFsPrincipal) {
-        String name = resolveId(uid, userNamespace.vespaUser(), userNamespace.vespaUserId());
+        String name = resolveId(uid, vespaUser.name(), vespaUser.uid());
         return new ContainerUserPrincipal(uid, name, baseFsPrincipal);
     }
     
     public ContainerGroupPrincipal groupPrincipal(int gid, GroupPrincipal baseFsPrincipal) {
-        String name = resolveId(gid, userNamespace.vespaGroup(), userNamespace.vespaGroupId());
+        String name = resolveId(gid, vespaUser.group(), vespaUser.gid());
         return new ContainerGroupPrincipal(gid, name, baseFsPrincipal);
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java
index 08e335f188a..4a26195dd3a 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java
@@ -93,7 +93,7 @@ public class ContainerTester implements AutoCloseable {
         NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) ->
                 NodeAgentContextImpl.builder(nodeSpec).acl(acl).fileSystem(fileSystem).build();
         nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator,
-                nodeAdmin, HOST_HOSTNAME, clock, flagSource);
+                nodeAdmin, HOST_HOSTNAME);
 
         loopThread = new Thread(() -> {
             nodeAdminStateUpdater.start();
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
index f9b0070a3d6..5436f84f467 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
@@ -3,8 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin;
 
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeType;
-import com.yahoo.test.ManualClock;
-import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
@@ -50,11 +48,9 @@ public class NodeAdminStateUpdaterTest {
     private final Orchestrator orchestrator = mock(Orchestrator.class);
     private final NodeAdmin nodeAdmin = mock(NodeAdmin.class);
     private final HostName hostHostname = HostName.from("basehost1.test.yahoo.com");
-    private final ManualClock clock = new ManualClock();
-    private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
 
     private final NodeAdminStateUpdater updater = spy(new NodeAdminStateUpdater(
-            nodeAgentContextFactory, nodeRepository, orchestrator, nodeAdmin, hostHostname, clock, flagSource));
+            nodeAgentContextFactory, nodeRepository, orchestrator, nodeAdmin, hostHostname));
 
 
     @Test
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java
index 73b59a17c37..bb02667a550 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java
@@ -11,7 +11,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
  */
 class UserNamespaceTest {
 
-    private final UserNamespace userNamespace = new UserNamespace(1000, 2000, "vespa", "users", 1000, 100);
+    private final UserNamespace userNamespace = new UserNamespace(1000, 2000);
 
     @Test
     public void translates_between_ids() {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java
index 4e85052a176..242a2458f07 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.node.admin.task.util.fs;
 
 import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.VespaUser;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
 import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.jupiter.api.Test;
@@ -24,8 +25,10 @@ class ContainerFileSystemTest {
 
     private final FileSystem fileSystem = TestFileSystem.create();
     private final UnixPath containerRootOnHost = new UnixPath(fileSystem.getPath("/data/storage/ctr1"));
-    private final UserNamespace userNamespace = new UserNamespace(10_000, 11_000, "vespa", "users", 1000, 100);
-    private final ContainerFileSystem containerFs = ContainerFileSystem.create(containerRootOnHost.createDirectories().toPath(), userNamespace);
+    private final UserNamespace userNamespace = new UserNamespace(10_000, 11_000);
+    private final VespaUser vespaUser = new VespaUser("vespa", "users", 1000, 100);
+    private final ContainerFileSystem containerFs = ContainerFileSystem.create(
+            containerRootOnHost.createDirectories().toPath(), userNamespace, vespaUser);
 
     @Test
     public void creates_files_and_directories_with_container_root_as_owner() throws IOException {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java
index 6bca8c2f0b1..795b3b198f5 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java
@@ -2,11 +2,18 @@
 package com.yahoo.vespa.hosted.node.admin.task.util.fs;
 
 import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.VespaUser;
 import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.function.Executable;
 
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+
 import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath.fromPathInContainer;
 import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath.fromPathOnHost;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -15,19 +22,13 @@ import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
 
-import java.io.IOException;
-import java.nio.file.FileSystem;
-import java.nio.file.Files;
-import java.nio.file.LinkOption;
-import java.nio.file.Path;
-
 /**
  * @author valerijf
  */
 class ContainerPathTest {
 
     private final FileSystem baseFs = TestFileSystem.create();
-    private final ContainerFileSystem containerFs = ContainerFileSystem.create(baseFs.getPath("/data/storage/ctr1"), mock(UserNamespace.class));
+    private final ContainerFileSystem containerFs = ContainerFileSystem.create(baseFs.getPath("/data/storage/ctr1"), mock(UserNamespace.class), mock(VespaUser.class));
 
     @Test
     public void create_new_container_path() {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java
index bc26cfa73f3..9a6e69ce27c 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java
@@ -2,14 +2,15 @@
 package com.yahoo.vespa.hosted.node.admin.task.util.fs;
 
 import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.VespaUser;
 import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
 import java.nio.file.attribute.UserPrincipalNotFoundException;
 
-import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal;
 import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal;
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
@@ -18,9 +19,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
  */
 class ContainerUserPrincipalLookupServiceTest {
 
-    private final UserNamespace userNamespace = new UserNamespace(10_000, 11_000, "vespa", "users", 1000, 100);
+    private final UserNamespace userNamespace = new UserNamespace(10_000, 11_000);
+    private final VespaUser vespaUser = new VespaUser("vespa", "users", 1000, 100);
     private final ContainerUserPrincipalLookupService userPrincipalLookupService =
-            new ContainerUserPrincipalLookupService(TestFileSystem.create().getUserPrincipalLookupService(), userNamespace);
+            new ContainerUserPrincipalLookupService(TestFileSystem.create().getUserPrincipalLookupService(), userNamespace, vespaUser);
 
     @Test
     public void correctly_resolves_ids() throws IOException {