diff options
author | Håkon Hallingstad <hakon@yahooinc.com> | 2023-04-21 16:46:37 +0200 |
---|---|---|
committer | Håkon Hallingstad <hakon@yahooinc.com> | 2023-04-21 16:46:37 +0200 |
commit | 83c4089539443def2125e63c8a50f5446e4b23bc (patch) | |
tree | b958fbbc1b1d5358c167409e1642164ef5ab0b59 /node-admin | |
parent | f3cd70d64b8287eb5d7e2406a882433b5faacd81 (diff) |
Start wireguard in a wireguard.scope cgroup
Diffstat (limited to 'node-admin')
3 files changed, 55 insertions, 18 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java index fef7f4bbd4c..78237b47eb1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.container; import com.yahoo.collections.Pair; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -27,12 +28,43 @@ public class CGroupV2 { private static final Logger logger = Logger.getLogger(CGroupV2.class.getName()); private static final String MAX = "max"; - public static final String VESPA_CGEXEC_PATH = "/opt/vespa/bin/vespa-cgexec"; - private final FileSystem fileSystem; + private final Path rootCgroupPath; public CGroupV2(FileSystem fileSystem) { - this.fileSystem = fileSystem; + this.rootCgroupPath = fileSystem.getPath("/sys/fs/cgroup"); + } + + /** + * Wraps {@code command} to ensure it is executed in the given cgroup. + * + * <p>WARNING: This method must be called only after vespa-cgexec has been installed.</p> + * + * @param cgroup The cgroup to execute the command in, e.g. system.slice/wireguard.scope. + * @param command The command to execute in the cgroup. + * @see #cgroupOf(ContainerId) + */ + public String[] wrapForExecutionIn(String cgroup, String... command) { + UnixPath path = new UnixPath(rootCgroupPath.resolve(cgroup).normalize()); + if (!path.toString().startsWith(rootCgroupPath + "/")) + throw new IllegalArgumentException("Invalid cgroup: " + cgroup); + // If more than one parent directory needs to be created, subtree_control needs to be fixed, somehow. + path.createDirectory(); // No-op if already exists + + String[] fullCommand = new String[3 + command.length]; + fullCommand[0] = Defaults.getDefaults().vespaHome() + "/bin/vespa-cgexec"; + fullCommand[1] = "-g"; + fullCommand[2] = cgroup; + System.arraycopy(command, 0, fullCommand, 3, command.length); + return fullCommand; + } + + /** + * Returns the cgroup directory of the Podman container relative the cgroup file system mount point, + * and which appears as the root cgroup within the container. + */ + public String cgroupOf(ContainerId containerId) { + return "machine.slice/libpod-" + containerId + ".scope/container"; } /** @@ -93,40 +125,40 @@ public class CGroupV2 { } public Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException { - return Files.readAllLines(cgroupRoot(containerId).resolve("cpu.stat")).stream() - .map(line -> line.split("\\s+")) - .filter(parts -> parts.length == 2) - .flatMap(parts -> CpuStatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1])))) - .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); + return Files.readAllLines(cgroupRootPath(containerId).resolve("cpu.stat")).stream() + .map(line -> line.split("\\s+")) + .filter(parts -> parts.length == 2) + .flatMap(parts -> CpuStatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1])))) + .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); } /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ public long memoryLimitInBytes(ContainerId containerId) throws IOException { - String limit = Files.readString(cgroupRoot(containerId).resolve("memory.max")).strip(); + String limit = Files.readString(cgroupRootPath(containerId).resolve("memory.max")).strip(); return MAX.equals(limit) ? -1L : Long.parseLong(limit); } /** @return The total amount of memory currently being used by the cgroup and its descendants. */ public long memoryUsageInBytes(ContainerId containerId) throws IOException { - return parseLong(cgroupRoot(containerId).resolve("memory.current")); + return parseLong(cgroupRootPath(containerId).resolve("memory.current")); } /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */ public long memoryCacheInBytes(ContainerId containerId) throws IOException { - return parseLong(cgroupRoot(containerId).resolve("memory.stat"), "file"); + return parseLong(cgroupRootPath(containerId).resolve("memory.stat"), "file"); } - private Path cgroupRoot(ContainerId containerId) { + private Path cgroupRootPath(ContainerId containerId) { // crun path, runc path is without the 'container' directory - return fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-" + containerId + ".scope/container"); + return rootCgroupPath.resolve(cgroupOf(containerId)); } private UnixPath cpuMaxPath(ContainerId containerId) { - return new UnixPath(cgroupRoot(containerId).resolve("cpu.max")); + return new UnixPath(cgroupRootPath(containerId).resolve("cpu.max")); } private UnixPath cpuWeightPath(ContainerId containerId) { - return new UnixPath(cgroupRoot(containerId).resolve("cpu.weight")); + return new UnixPath(cgroupRootPath(containerId).resolve("cpu.weight")); } private static boolean writeCGroupsValue(NodeAgentContext context, UnixPath unixPath, String value) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java index 073e4263492..858b3d647fc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; +import com.yahoo.vespa.hosted.node.admin.container.ContainerId; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; /** @@ -9,6 +10,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; */ public interface ContainerWireguardTask { - void converge(NodeAgentContext context); + void converge(NodeAgentContext context, ContainerId containerId); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index f2f690106fa..7c84afc8397 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -509,7 +509,8 @@ public class NodeAgentImpl implements NodeAgent { // TODO: this is a workaround for restarting wireguard as early as possible after host-admin has been down. var runOrdinaryWireguardTasks = true; if (container.isPresent() && container.get().state().isRunning()) { - wireguardTasks.forEach(task -> task.converge(context)); + Optional<Container> finalContainer = container; + wireguardTasks.forEach(task -> task.converge(context, finalContainer.get().id())); runOrdinaryWireguardTasks = false; } @@ -530,7 +531,10 @@ public class NodeAgentImpl implements NodeAgent { } aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); - if (runOrdinaryWireguardTasks) wireguardTasks.forEach(task -> task.converge(context)); + if (runOrdinaryWireguardTasks) { + Optional<Container> finalContainer = container; + wireguardTasks.forEach(task -> task.converge(context, finalContainer.get().id())); + } startServicesIfNeeded(context); resumeNodeIfNeeded(context); if (healthChecker.isPresent()) { |