Start wireguard in a wireguard.scope cgroup

3 files changed, 55 insertions, 18 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java
index fef7f4bbd4c..78237b47eb1 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.node.admin.container;
 
 import com.yahoo.collections.Pair;
+import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
 
@@ -27,12 +28,43 @@ public class CGroupV2 {
 
     private static final Logger logger = Logger.getLogger(CGroupV2.class.getName());
     private static final String MAX = "max";
-    public static final String VESPA_CGEXEC_PATH = "/opt/vespa/bin/vespa-cgexec";
 
-    private final FileSystem fileSystem;
+    private final Path rootCgroupPath;
 
     public CGroupV2(FileSystem fileSystem) {
-        this.fileSystem = fileSystem;
+        this.rootCgroupPath = fileSystem.getPath("/sys/fs/cgroup");
+    }
+
+    /**
+     * Wraps {@code command} to ensure it is executed in the given cgroup.
+     *
+     * <p>WARNING: This method must be called only after vespa-cgexec has been installed.</p>
+     *
+     * @param cgroup  The cgroup to execute the command in, e.g. system.slice/wireguard.scope.
+     * @param command The command to execute in the cgroup.
+     * @see #cgroupOf(ContainerId)
+     */
+    public String[] wrapForExecutionIn(String cgroup, String... command) {
+        UnixPath path = new UnixPath(rootCgroupPath.resolve(cgroup).normalize());
+        if (!path.toString().startsWith(rootCgroupPath + "/"))
+            throw new IllegalArgumentException("Invalid cgroup: " + cgroup);
+        // If more than one parent directory needs to be created, subtree_control needs to be fixed, somehow.
+        path.createDirectory();  // No-op if already exists
+
+        String[] fullCommand = new String[3 + command.length];
+        fullCommand[0] = Defaults.getDefaults().vespaHome() + "/bin/vespa-cgexec";
+        fullCommand[1] = "-g";
+        fullCommand[2] = cgroup;
+        System.arraycopy(command, 0, fullCommand, 3, command.length);
+        return fullCommand;
+    }
+
+    /**
+     * Returns the cgroup directory of the Podman container relative the cgroup file system mount point,
+     * and which appears as the root cgroup within the container.
+     */
+    public String cgroupOf(ContainerId containerId) {
+        return "machine.slice/libpod-" + containerId + ".scope/container";
     }
 
     /**
@@ -93,40 +125,40 @@ public class CGroupV2 {
     }
 
     public Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException {
-        return Files.readAllLines(cgroupRoot(containerId).resolve("cpu.stat")).stream()
-                .map(line -> line.split("\\s+"))
-                .filter(parts -> parts.length == 2)
-                .flatMap(parts -> CpuStatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1]))))
-                .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond));
+        return Files.readAllLines(cgroupRootPath(containerId).resolve("cpu.stat")).stream()
+                    .map(line -> line.split("\\s+"))
+                    .filter(parts -> parts.length == 2)
+                    .flatMap(parts -> CpuStatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1]))))
+                    .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond));
     }
 
     /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */
     public long memoryLimitInBytes(ContainerId containerId) throws IOException {
-        String limit = Files.readString(cgroupRoot(containerId).resolve("memory.max")).strip();
+        String limit = Files.readString(cgroupRootPath(containerId).resolve("memory.max")).strip();
         return MAX.equals(limit) ? -1L : Long.parseLong(limit);
     }
 
     /** @return The total amount of memory currently being used by the cgroup and its descendants. */
     public long memoryUsageInBytes(ContainerId containerId) throws IOException {
-        return parseLong(cgroupRoot(containerId).resolve("memory.current"));
+        return parseLong(cgroupRootPath(containerId).resolve("memory.current"));
     }
 
     /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */
     public long memoryCacheInBytes(ContainerId containerId) throws IOException {
-        return parseLong(cgroupRoot(containerId).resolve("memory.stat"), "file");
+        return parseLong(cgroupRootPath(containerId).resolve("memory.stat"), "file");
     }
 
-    private Path cgroupRoot(ContainerId containerId) {
+    private Path cgroupRootPath(ContainerId containerId) {
         // crun path, runc path is without the 'container' directory
-        return fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-" + containerId + ".scope/container");
+        return rootCgroupPath.resolve(cgroupOf(containerId));
     }
 
     private UnixPath cpuMaxPath(ContainerId containerId) {
-        return new UnixPath(cgroupRoot(containerId).resolve("cpu.max"));
+        return new UnixPath(cgroupRootPath(containerId).resolve("cpu.max"));
     }
 
     private UnixPath cpuWeightPath(ContainerId containerId) {
-        return new UnixPath(cgroupRoot(containerId).resolve("cpu.weight"));
+        return new UnixPath(cgroupRootPath(containerId).resolve("cpu.weight"));
     }
 
     private static boolean writeCGroupsValue(NodeAgentContext context, UnixPath unixPath, String value) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java
index 073e4263492..858b3d647fc 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java
@@ -1,5 +1,6 @@
 package com.yahoo.vespa.hosted.node.admin.maintenance;
 
+import com.yahoo.vespa.hosted.node.admin.container.ContainerId;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 
 /**
@@ -9,6 +10,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
  */
 public interface ContainerWireguardTask {
 
-    void converge(NodeAgentContext context);
+    void converge(NodeAgentContext context, ContainerId containerId);
 
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index f2f690106fa..7c84afc8397 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -509,7 +509,8 @@ public class NodeAgentImpl implements NodeAgent {
                 // TODO: this is a workaround for restarting wireguard as early as possible after host-admin has been down.
                 var runOrdinaryWireguardTasks = true;
                 if (container.isPresent() && container.get().state().isRunning()) {
-                    wireguardTasks.forEach(task -> task.converge(context));
+                    Optional<Container> finalContainer = container;
+                    wireguardTasks.forEach(task -> task.converge(context, finalContainer.get().id()));
                     runOrdinaryWireguardTasks = false;
                 }
 
@@ -530,7 +531,10 @@ public class NodeAgentImpl implements NodeAgent {
                 }
 
                 aclMaintainer.ifPresent(maintainer -> maintainer.converge(context));
-                if (runOrdinaryWireguardTasks) wireguardTasks.forEach(task -> task.converge(context));
+                if (runOrdinaryWireguardTasks) {
+                    Optional<Container> finalContainer = container;
+                    wireguardTasks.forEach(task -> task.converge(context, finalContainer.get().id()));
+                }
                 startServicesIfNeeded(context);
                 resumeNodeIfNeeded(context);
                 if (healthChecker.isPresent()) {