diff options
author | Håkon Hallingstad <hakon@yahooinc.com> | 2023-04-26 15:07:33 +0200 |
---|---|---|
committer | Håkon Hallingstad <hakon@yahooinc.com> | 2023-04-26 15:07:33 +0200 |
commit | 75e261266c3629e4343f40f1aa26fc2dc02c9aa3 (patch) | |
tree | 92167ef8b2372f2d5f20efffb3a91200dc838369 /node-admin | |
parent | dbe38f35cb1d26a146b41cf644280007b550840e (diff) |
Improve cgroup modeling
Diffstat (limited to 'node-admin')
12 files changed, 410 insertions, 253 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/ControlGroup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/ControlGroup.java new file mode 100644 index 00000000000..2fa055f2151 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/ControlGroup.java @@ -0,0 +1,89 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.hosted.node.admin.container.ContainerId; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; + +import java.nio.file.FileSystem; +import java.nio.file.Path; + +/** + * Represents a cgroup directory in the control group v2 hierarchy, see + * <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html">Control Group v2</a>. + * + * @author hakonhall + */ +public class ControlGroup { + private final Path root; + private final Path relativePath; + + public static ControlGroup root(FileSystem fileSystem) { + return new ControlGroup(fileSystem.getPath("/sys/fs/cgroup"), fileSystem.getPath("")); + } + + private ControlGroup(Path root, Path relativePath) { + this.root = root.normalize(); + this.relativePath = this.root.relativize(this.root.resolve(relativePath).normalize()); + if (this.relativePath.toString().equals("..") || this.relativePath.toString().startsWith("../")) { + throw new IllegalArgumentException("Invalid cgroup relative path: " + relativePath); + } + } + + /** + * Resolve the given path against the path of this cgroup, and return the resulting cgroup. + * If the given path is absolute, it is resolved against the root of the cgroup hierarchy. + */ + public ControlGroup resolve(String path) { + Path effectivePath = fileSystem().getPath(path); + if (effectivePath.isAbsolute()) { + return new ControlGroup(root, fileSystem().getPath("/").relativize(effectivePath)); + } else { + return new ControlGroup(root, relativePath.resolve(path)); + } + } + + /** Returns the parent ControlGroup. */ + public ControlGroup resolveParent() { return new ControlGroup(root, relativePath.getParent()); } + + /** Returns the ControlGroup of a system service, e.g. vespa-host-admin. */ + public ControlGroup resolveSystemService(String name) { return resolve("/system.slice").resolve(serviceNameOf(name)); } + + /** Returns the root ControlGroup of the given Podman container. */ + public ControlGroup resolveContainer(ContainerId containerId) { return resolve("/machine.slice/libpod-" + containerId + ".scope/container"); } + + /** Returns the ControlGroup of a system service in the given Podman container. */ + public ControlGroup resolveContainerSystemService(ContainerId containerId, String name) { return resolveContainer(containerId).resolve("system.slice").resolve(serviceNameOf(name)); } + + /** Returns the absolute path to this cgroup. */ + public Path path() { return root.resolve(relativePath); } + + /** Returns the absolute UnixPath to this cgroup. */ + public UnixPath unixPath() { return new UnixPath(path()); } + + /** Returns the CPU controller of this ControlGroup. */ + public CpuController cpu() { return new CpuController(this); } + + /** Returns the memory controller of this ControlGroup. */ + public MemoryController memory() { return new MemoryController(this); } + + /** + * Wraps {@code command} to ensure it is executed in this cgroup. + * + * <p>WARNING: This method must be called only after vespa-cgexec has been installed.</p> + */ + public String[] wrapCommandForExecutionInCgroup(String... command) { + String[] fullCommand = new String[3 + command.length]; + fullCommand[0] = Defaults.getDefaults().vespaHome() + "/bin/vespa-cgexec"; + fullCommand[1] = "-g"; + fullCommand[2] = relativePath.toString(); + System.arraycopy(command, 0, fullCommand, 3, command.length); + return fullCommand; + } + + private static String serviceNameOf(String name) { + return name.indexOf('.') == -1 ? name + ".service" : name; + } + + private FileSystem fileSystem() { return root.getFileSystem(); } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java new file mode 100644 index 00000000000..9b6f0942c2a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java @@ -0,0 +1,132 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; + +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static java.lang.Integer.parseInt; + +/** + * Represents a cgroup v2 CPU controller, i.e. all files named cpu.* + * + * @author hakonhall + */ +public class CpuController { + private static final Logger logger = Logger.getLogger(CpuController.class.getName()); + + private final ControlGroup cgroup; + + CpuController(ControlGroup cgroup) { + this.cgroup = cgroup; + } + + /** + * The maximum bandwidth limit of the format "QUOTA PERIOD", which indicates that the cgroup may consume + * up to QUOTA in each PERIOD duration. A quota of "max" indicates no limit. + */ + public record Max(Size quota, int period) { + public String toFileContent() { return quota + " " + period; } + } + + /** + * Returns the maximum CPU usage, or empty if cgroup is not found. + * + * @see Max + */ + public Optional<Max> readMax() { + return cgroup.unixPath() + .resolve("cpu.max") + .readUtf8FileIfExists() + .map(content -> { + String[] parts = content.strip().split(" "); + return new Max(Size.from(parts[0]), parseInt(parts[1])); + }); + } + + /** + * Update CPU quota and period for the given container ID. Set quota to -1 value for unlimited. + * + * @see #readMax() + * @see Max + */ + public boolean updateMax(NodeAgentContext context, int quota, int period) { + Max max = new Max(quota < 0 ? Size.max() : Size.from(quota), period); + return convergeFileContent(context, "cpu.max", max.toFileContent()); + } + + /** @return The weight in the range [1, 10000], or empty if not found. */ + private Optional<Integer> readWeight() { + return cgroup.unixPath() + .resolve("cpu.weight") + .readUtf8FileIfExists() + .map(content -> parseInt(content.strip())); + } + + /** @return The number of shares allocated to this cgroup for purposes of CPU time scheduling, or empty if not found. */ + public Optional<Integer> readShares() { + return readWeight().map(CpuController::weightToShares); + } + + public boolean updateShares(NodeAgentContext context, int shares) { + return convergeFileContent(context, "cpu.weight", Integer.toString(sharesToWeight(shares))); + } + + // Must be same as in crun: https://github.com/containers/crun/blob/72c6e60ade0e4716fe2d8353f0d97d72cc8d1510/src/libcrun/cgroup.c#L3061 + // TODO: Migrate to weights + public static int sharesToWeight(int shares) { return (int) (1 + ((shares - 2L) * 9999) / 262142); } + public static int weightToShares(int weight) { return (int) (2 + ((weight - 1L) * 262142) / 9999); } + + private boolean convergeFileContent(NodeAgentContext context, String filename, String content) { + UnixPath path = cgroup.unixPath().resolve(filename); + String currentContent = path.readUtf8File().strip(); + if (currentContent.equals(content)) return false; + + context.recordSystemModification(logger, "Updating " + path + " from " + currentContent + " to " + content); + path.writeUtf8File(content); + return true; + } + + public enum StatField { + TOTAL_USAGE_USEC("usage_usec"), + USER_USAGE_USEC("user_usec"), + SYSTEM_USAGE_USEC("system_usec"), + TOTAL_PERIODS("nr_periods"), + THROTTLED_PERIODS("nr_throttled"), + THROTTLED_TIME_USEC("throttled_usec"); + + private final String name; + + StatField(String name) { + this.name = name; + } + + long parseValue(String value) { + return Long.parseLong(value); + } + + static Optional<StatField> fromField(String fieldName) { + return Arrays.stream(values()) + .filter(field -> fieldName.equals(field.name)) + .findFirst(); + } + } + + public Map<StatField, Long> readStats() { + return cgroup.unixPath() + .resolve("cpu.stat") + .readAllLines() + .stream() + .map(line -> line.split("\\s+")) + .filter(parts -> parts.length == 2) + .flatMap(parts -> StatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1])))) + .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java new file mode 100644 index 00000000000..a1df9f20471 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; + +/** + * Represents a cgroup v2 memory controller, i.e. all files named memory.* + * + * @author hakonhall + */ +public class MemoryController { + private final ControlGroup cgroup; + + MemoryController(ControlGroup cgroup) { + this.cgroup = cgroup; + } + + /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ + public Size readMax() { + return Size.from(cgroup.unixPath().resolve("memory.max").readUtf8File().strip()); + } + + /** @return The total amount of memory currently being used by the cgroup and its descendants, in bytes. */ + public Size readCurrent() { + return Size.from(cgroup.unixPath().resolve("memory.current").readUtf8File().strip()); + } + + /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */ + public Size readFileSystemCache() { + return Size.from(readField(cgroup.unixPath().resolve("memory.stat"), "file")); + } + + private static String readField(UnixPath path, String fieldName) { + return path.readAllLines() + .stream() + .map(line -> line.split("\\s+")) + .filter(fields -> fields.length == 2) + .filter(fields -> fieldName.equals(fields[0])) + .map(fields -> fields[1]) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No such field: " + fieldName)); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java new file mode 100644 index 00000000000..c03b84fe579 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import java.util.Objects; + +/** + * Represents a number of bytes or possibly "max". + * + * @author hakonhall + */ +public class Size { + private static final String MAX = "max"; + + private final boolean max; + private final long value; + + public static Size max() { + return new Size(true, 0); + } + + public static Size from(long value) { + return new Size(false, value); + } + + public static Size from(String value) { + return value.equals(MAX) ? new Size(true, 0) : new Size(false, Long.parseLong(value)); + } + + private Size(boolean max, long value) { + this.max = max; + this.value = value; + } + + public boolean isMax() { + return max; + } + + /** Returns the value, i.e. the number of "bytes" if applicable. Throws if this is max. */ + public long value() { + if (max) throw new IllegalStateException("Value is max"); + return value; + } + + @Override + public String toString() { return max ? MAX : Long.toString(value); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Size size = (Size) o; + return max == size.max && value == size.value; + } + + @Override + public int hashCode() { + return Objects.hash(max, value); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java deleted file mode 100644 index 3cb34e066ff..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.collections.Pair; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Read and write interface to the cgroup v2 of a Podman container. - * - * @see <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html">CGroups V2</a> - * @author freva - */ -public class CGroupV2 { - - private static final Logger logger = Logger.getLogger(CGroupV2.class.getName()); - private static final String MAX = "max"; - - private final Path rootCgroupPath; - - public CGroupV2(FileSystem fileSystem) { - this.rootCgroupPath = fileSystem.getPath("/sys/fs/cgroup"); - } - - /** - * Wraps {@code command} to ensure it is executed in the given cgroup. - * - * <p>WARNING: This method must be called only after vespa-cgexec has been installed.</p> - * - * @param cgroup The cgroup to execute the command in, e.g. /sys/fs/cgroup/system.slice/wireguard.scope. - * @param command The command to execute in the cgroup. - * @see #cgroupRootPath() - * @see #cgroupPath(ContainerId) - */ - public String[] wrapForExecutionIn(Path cgroup, String... command) { - String[] fullCommand = new String[3 + command.length]; - fullCommand[0] = Defaults.getDefaults().vespaHome() + "/bin/vespa-cgexec"; - fullCommand[1] = "-g"; - fullCommand[2] = cgroup.toString(); - System.arraycopy(command, 0, fullCommand, 3, command.length); - return fullCommand; - } - - /** - * Returns quota and period values used for CPU scheduling. This serves as hard cap on CPU usage by allowing - * the CGroupV2 to use up to {@code quota} each {@code period}. If uncapped, quota will be negative. - * - * @param containerId full container ID. - * @return CPU quota and period for the given container. Empty if CGroupV2 for this container is not found. - */ - public Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId) { - return cpuMaxPath(containerId).readUtf8FileIfExists() - .map(s -> { - String[] parts = s.strip().split(" "); - return new Pair<>(MAX.equals(parts[0]) ? -1 : Integer.parseInt(parts[0]), Integer.parseInt(parts[1])); - }); - } - - /** @return number of shares allocated to this CGroupV2 for purposes of CPU time scheduling, empty if CGroupV2 not found */ - public OptionalInt cpuShares(ContainerId containerId) { - return cpuWeightPath(containerId).readUtf8FileIfExists() - .map(s -> OptionalInt.of(weightToShares(Integer.parseInt(s.strip())))) - .orElseGet(OptionalInt::empty); - } - - /** Update CPU quota and period for the given container ID, set quota to -1 value for unlimited */ - public boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs) { - String wanted = String.format("%s %d", cpuQuotaUs < 0 ? MAX : cpuQuotaUs, periodUs); - return writeCGroupsValue(context, cpuMaxPath(containerId), wanted); - } - - public boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares) { - return writeCGroupsValue(context, cpuWeightPath(containerId), Integer.toString(sharesToWeight(shares))); - } - - enum CpuStatField { - TOTAL_USAGE_USEC("usage_usec"), - USER_USAGE_USEC("user_usec"), - SYSTEM_USAGE_USEC("system_usec"), - TOTAL_PERIODS("nr_periods"), - THROTTLED_PERIODS("nr_throttled"), - THROTTLED_TIME_USEC("throttled_usec"); - - private final String name; - - CpuStatField(String name) { - this.name = name; - } - - long parseValue(String value) { - return Long.parseLong(value); - } - - static Optional<CpuStatField> fromField(String fieldName) { - return Arrays.stream(values()) - .filter(field -> fieldName.equals(field.name)) - .findFirst(); - } - } - - public Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException { - return Files.readAllLines(cgroupPath(containerId).resolve("cpu.stat")).stream() - .map(line -> line.split("\\s+")) - .filter(parts -> parts.length == 2) - .flatMap(parts -> CpuStatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1])))) - .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); - } - - /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ - public long memoryLimitInBytes(ContainerId containerId) throws IOException { - String limit = Files.readString(cgroupPath(containerId).resolve("memory.max")).strip(); - return MAX.equals(limit) ? -1L : Long.parseLong(limit); - } - - /** @return The total amount of memory currently being used by the cgroup and its descendants. */ - public long memoryUsageInBytes(ContainerId containerId) throws IOException { - return parseLong(cgroupPath(containerId).resolve("memory.current")); - } - - /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */ - public long memoryCacheInBytes(ContainerId containerId) throws IOException { - return parseLong(cgroupPath(containerId).resolve("memory.stat"), "file"); - } - - /** Returns the cgroup v2 mount point path (/sys/fs/cgroup). */ - public Path cgroupRootPath() { - return rootCgroupPath; - } - - /** Returns the cgroup directory of the Podman container, and which appears as the root cgroup within the container. */ - public Path cgroupPath(ContainerId containerId) { - // crun path, runc path is without the 'container' directory - return rootCgroupPath.resolve("machine.slice/libpod-" + containerId + ".scope/container"); - } - - private UnixPath cpuMaxPath(ContainerId containerId) { - return new UnixPath(cgroupPath(containerId).resolve("cpu.max")); - } - - private UnixPath cpuWeightPath(ContainerId containerId) { - return new UnixPath(cgroupPath(containerId).resolve("cpu.weight")); - } - - private static boolean writeCGroupsValue(NodeAgentContext context, UnixPath unixPath, String value) { - String currentValue = unixPath.readUtf8File().strip(); - if (currentValue.equals(value)) return false; - - context.recordSystemModification(logger, "Updating " + unixPath + " from " + currentValue + " to " + value); - unixPath.writeUtf8File(value); - return true; - } - - // Must be same as in crun: https://github.com/containers/crun/blob/72c6e60ade0e4716fe2d8353f0d97d72cc8d1510/src/libcrun/cgroup.c#L3061 - static int sharesToWeight(int shares) { return (int) (1 + ((shares - 2L) * 9999) / 262142); } - static int weightToShares(int weight) { return (int) (2 + ((weight - 1L) * 262142) / 9999); } - - static long parseLong(Path path) throws IOException { - return Long.parseLong(Files.readString(path).trim()); - } - - static long parseLong(Path path, String fieldName) throws IOException { - return parseLong(Files.readAllLines(path), fieldName); - } - - static long parseLong(List<String> lines, String fieldName) { - for (String line : lines) { - String[] fields = line.split("\\s+"); - if (fields.length != 2) - throw new IllegalArgumentException("Expected line on the format 'key value', got: '" + line + "'"); - - if (fieldName.equals(fields[0])) return Long.parseLong(fields[1]); - } - throw new IllegalArgumentException("No such field: " + fieldName); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index ce2a6bb22ac..101b90203f6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.container; import com.yahoo.config.provision.DockerImage; +import com.yahoo.vespa.hosted.node.admin.cgroup.ControlGroup; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.container.image.ContainerImageDownloader; import com.yahoo.vespa.hosted.node.admin.container.image.ContainerImagePruner; @@ -33,7 +34,7 @@ public class ContainerOperations { private final ContainerImagePruner imagePruner; private final ContainerStatsCollector containerStatsCollector; - public ContainerOperations(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem) { + public ContainerOperations(ContainerEngine containerEngine, ControlGroup cgroup, FileSystem fileSystem) { this.containerEngine = Objects.requireNonNull(containerEngine); this.imageDownloader = new ContainerImageDownloader(containerEngine); this.imagePruner = new ContainerImagePruner(containerEngine, Clock.systemUTC()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java index 870809123a9..adbfc52d397 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java @@ -1,6 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.container; +import com.yahoo.vespa.hosted.node.admin.cgroup.ControlGroup; +import com.yahoo.vespa.hosted.node.admin.cgroup.CpuController; +import com.yahoo.vespa.hosted.node.admin.cgroup.Size; +import com.yahoo.vespa.hosted.node.admin.cgroup.MemoryController; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; @@ -27,18 +31,18 @@ import java.util.stream.Stream; class ContainerStatsCollector { private final ContainerEngine containerEngine; - private final CGroupV2 cgroup; private final FileSystem fileSystem; + private final ControlGroup rootCgroup; private final int onlineCpus; - ContainerStatsCollector(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem) { - this(containerEngine, cgroup, fileSystem, Runtime.getRuntime().availableProcessors()); + ContainerStatsCollector(ContainerEngine containerEngine, ControlGroup rootCgroup, FileSystem fileSystem) { + this(containerEngine, rootCgroup, fileSystem, Runtime.getRuntime().availableProcessors()); } - ContainerStatsCollector(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem, int onlineCpus) { + ContainerStatsCollector(ContainerEngine containerEngine, ControlGroup rootCgroup, FileSystem fileSystem, int onlineCpus) { this.containerEngine = Objects.requireNonNull(containerEngine); - this.cgroup = Objects.requireNonNull(cgroup); this.fileSystem = Objects.requireNonNull(fileSystem); + this.rootCgroup = Objects.requireNonNull(rootCgroup); this.onlineCpus = onlineCpus; } @@ -52,6 +56,10 @@ class ContainerStatsCollector { return Optional.of(new ContainerStats(networkStats, memoryStats, cpuStats, gpuStats)); } catch (NoSuchFileException ignored) { return Optional.empty(); // Container disappeared while we collected stats + } catch (UncheckedIOException e) { + if (e.getCause() != null && e.getCause() instanceof NoSuchFileException) + return Optional.empty(); + throw e; } catch (IOException e) { throw new UncheckedIOException(e); } @@ -83,21 +91,22 @@ class ContainerStatsCollector { } private ContainerStats.CpuStats collectCpuStats(ContainerId containerId) throws IOException { - Map<CGroupV2.CpuStatField, Long> cpuStats = cgroup.cpuStats(containerId); + Map<CpuController.StatField, Long> cpuStats = rootCgroup.resolveContainer(containerId).cpu().readStats(); return new ContainerStats.CpuStats(onlineCpus, systemCpuUsage(), - cpuStats.get(CGroupV2.CpuStatField.TOTAL_USAGE_USEC), - cpuStats.get(CGroupV2.CpuStatField.SYSTEM_USAGE_USEC), - cpuStats.get(CGroupV2.CpuStatField.THROTTLED_TIME_USEC), - cpuStats.get(CGroupV2.CpuStatField.TOTAL_PERIODS), - cpuStats.get(CGroupV2.CpuStatField.THROTTLED_PERIODS)); + cpuStats.get(CpuController.StatField.TOTAL_USAGE_USEC), + cpuStats.get(CpuController.StatField.SYSTEM_USAGE_USEC), + cpuStats.get(CpuController.StatField.THROTTLED_TIME_USEC), + cpuStats.get(CpuController.StatField.TOTAL_PERIODS), + cpuStats.get(CpuController.StatField.THROTTLED_PERIODS)); } private ContainerStats.MemoryStats collectMemoryStats(ContainerId containerId) throws IOException { - long memoryLimitInBytes = cgroup.memoryLimitInBytes(containerId); - long memoryUsageInBytes = cgroup.memoryUsageInBytes(containerId); - long cachedInBytes = cgroup.memoryCacheInBytes(containerId); - return new ContainerStats.MemoryStats(cachedInBytes, memoryUsageInBytes, memoryLimitInBytes); + MemoryController memoryController = rootCgroup.resolveContainer(containerId).memory(); + Size max = memoryController.readMax(); + long memoryUsageInBytes = memoryController.readCurrent().value(); + long cachedInBytes = memoryController.readFileSystemCache().value(); + return new ContainerStats.MemoryStats(cachedInBytes, memoryUsageInBytes, max.isMax() ? -1 : max.value()); } private ContainerStats.NetworkStats collectNetworkStats(String iface, int containerPid) throws IOException { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index fbef3def446..e6786b37b93 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -23,6 +23,7 @@ import java.nio.file.attribute.PosixFilePermissions; import java.nio.file.attribute.UserPrincipal; import java.nio.file.attribute.UserPrincipalLookupService; import java.time.Instant; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -99,6 +100,10 @@ public class UnixPath { } } + public List<String> readAllLines() { + return uncheck(() -> Files.readAllLines(path)); + } + public UnixPath writeUtf8File(String content, OpenOption... options) { return writeBytes(content.getBytes(StandardCharsets.UTF_8), options); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/ControlGroupTest.java index 789f31f75c6..7c040f4ed06 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/ControlGroupTest.java @@ -1,7 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; -import com.yahoo.collections.Pair; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import com.yahoo.vespa.hosted.node.admin.container.ContainerId; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -12,16 +14,15 @@ import java.io.IOException; import java.nio.file.FileSystem; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; - -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.USER_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.sharesToWeight; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.weightToShares; + +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.sharesToWeight; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.weightToShares; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -29,47 +30,48 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author freva */ -public class CGroupV2Test { +public class ControlGroupTest { private static final ContainerId containerId = new ContainerId("4aec78cc"); private final FileSystem fileSystem = TestFileSystem.create(); - private final CGroupV2 cgroup = new CGroupV2(fileSystem); + private final ControlGroup containerCgroup = ControlGroup.root(fileSystem).resolveContainer(containerId); + private final CpuController containerCpu = containerCgroup.cpu(); private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); private final UnixPath cgroupRoot = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-4aec78cc.scope/container")).createDirectories(); @Test public void updates_cpu_quota_and_period() { - assertEquals(Optional.empty(), cgroup.cpuQuotaPeriod(containerId)); + assertEquals(Optional.empty(), containerCgroup.cpu().readMax()); cgroupRoot.resolve("cpu.max").writeUtf8File("max 100000\n"); - assertEquals(Optional.of(new Pair<>(-1, 100000)), cgroup.cpuQuotaPeriod(containerId)); + assertEquals(Optional.of(new CpuController.Max(Size.max(), 100000)), containerCpu.readMax()); cgroupRoot.resolve("cpu.max").writeUtf8File("456 123456\n"); - assertEquals(Optional.of(new Pair<>(456, 123456)), cgroup.cpuQuotaPeriod(containerId)); + assertEquals(Optional.of(new CpuController.Max(Size.from(456), 123456)), containerCpu.readMax()); - assertFalse(cgroup.updateCpuQuotaPeriod(context, containerId, 456, 123456)); + containerCgroup.cpu().updateMax(context, 456, 123456); - assertTrue(cgroup.updateCpuQuotaPeriod(context, containerId, 654, 123456)); - assertEquals(Optional.of(new Pair<>(654, 123456)), cgroup.cpuQuotaPeriod(containerId)); + assertTrue(containerCgroup.cpu().updateMax(context, 654, 123456)); + assertEquals(Optional.of(new CpuController.Max(Size.from(654), 123456)), containerCpu.readMax()); assertEquals("654 123456", cgroupRoot.resolve("cpu.max").readUtf8File()); - assertTrue(cgroup.updateCpuQuotaPeriod(context, containerId, -1, 123456)); - assertEquals(Optional.of(new Pair<>(-1, 123456)), cgroup.cpuQuotaPeriod(containerId)); + assertTrue(containerCgroup.cpu().updateMax(context, -1, 123456)); + assertEquals(Optional.of(new CpuController.Max(Size.max(), 123456)), containerCpu.readMax()); assertEquals("max 123456", cgroupRoot.resolve("cpu.max").readUtf8File()); } @Test public void updates_cpu_shares() { - assertEquals(OptionalInt.empty(), cgroup.cpuShares(containerId)); + assertEquals(Optional.empty(), containerCgroup.cpu().readShares()); cgroupRoot.resolve("cpu.weight").writeUtf8File("1\n"); - assertEquals(OptionalInt.of(2), cgroup.cpuShares(containerId)); + assertEquals(Optional.of(2), containerCgroup.cpu().readShares()); - assertFalse(cgroup.updateCpuShares(context, containerId, 2)); + assertFalse(containerCgroup.cpu().updateShares(context, 2)); - assertTrue(cgroup.updateCpuShares(context, containerId, 12345)); - assertEquals(OptionalInt.of(12323), cgroup.cpuShares(containerId)); + assertTrue(containerCgroup.cpu().updateShares(context, 12345)); + assertEquals(Optional.of(12323), containerCgroup.cpu().readShares()); } @Test @@ -82,16 +84,16 @@ public class CGroupV2Test { "throttled_usec 14256\n"); assertEquals(Map.of(TOTAL_USAGE_USEC, 17794243L, USER_USAGE_USEC, 16099205L, SYSTEM_USAGE_USEC, 1695038L, - TOTAL_PERIODS, 12465L, THROTTLED_PERIODS, 25L, THROTTLED_TIME_USEC, 14256L), cgroup.cpuStats(containerId)); + TOTAL_PERIODS, 12465L, THROTTLED_PERIODS, 25L, THROTTLED_TIME_USEC, 14256L), containerCgroup.cpu().readStats()); } @Test public void reads_memory_metrics() throws IOException { cgroupRoot.resolve("memory.current").writeUtf8File("2525093888\n"); - assertEquals(2525093888L, cgroup.memoryUsageInBytes(containerId)); + assertEquals(2525093888L, containerCgroup.memory().readCurrent().value()); cgroupRoot.resolve("memory.max").writeUtf8File("4322885632\n"); - assertEquals(4322885632L, cgroup.memoryLimitInBytes(containerId)); + assertEquals(4322885632L, containerCgroup.memory().readMax().value()); cgroupRoot.resolve("memory.stat").writeUtf8File("anon 3481600\n" + "file 69206016\n" + @@ -102,7 +104,7 @@ public class CGroupV2Test { "shmem 8380416\n" + "file_mapped 1081344\n" + "file_dirty 135168\n"); - assertEquals(69206016L, cgroup.memoryCacheInBytes(containerId)); + assertEquals(69206016L, containerCgroup.memory().readFileSystemCache().value()); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java index 701dd33cf55..5dbc2128051 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.container; import com.yahoo.config.provision.DockerImage; +import com.yahoo.vespa.hosted.node.admin.cgroup.ControlGroup; import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; @@ -11,9 +12,11 @@ import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; /** * @author mpolden @@ -23,7 +26,7 @@ public class ContainerOperationsTest { private final TestTaskContext context = new TestTaskContext(); private final ContainerEngineMock containerEngine = new ContainerEngineMock(); private final FileSystem fileSystem = TestFileSystem.create(); - private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, new CGroupV2(fileSystem), fileSystem); + private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, mock(ControlGroup.class), fileSystem); @Test void no_managed_containers_running() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java index 72c5d016a47..59110ef4bd2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.container; +import com.yahoo.vespa.hosted.node.admin.cgroup.ControlGroup; +import com.yahoo.vespa.hosted.node.admin.cgroup.Size; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; @@ -8,6 +10,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; +import org.mockito.Answers; import java.io.IOException; import java.nio.file.FileSystem; @@ -17,12 +20,12 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.USER_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; @@ -37,11 +40,10 @@ public class ContainerStatsCollectorTest { private final TestTerminal testTerminal = new TestTerminal(); private final ContainerEngineMock containerEngine = new ContainerEngineMock(testTerminal); private final FileSystem fileSystem = TestFileSystem.create(); - private final CGroupV2 cgroup = mock(CGroupV2.class); + private final ControlGroup cgroup = mock(ControlGroup.class, Answers.RETURNS_DEEP_STUBS); private final NodeAgentContext context = NodeAgentContextImpl.builder(NodeSpec.Builder.testSpec("c1").build()) .fileSystem(TestFileSystem.create()) .build(); - @Test void collect() throws Exception { ContainerStatsCollector collector = new ContainerStatsCollector(containerEngine, cgroup, fileSystem, 24); @@ -92,17 +94,17 @@ public class ContainerStatsCollectorTest { " eth0: 22280813 118083 3 4 0 0 0 0 19859383 115415 5 6 0 0 0 0\n"); } - private void mockMemoryStats(ContainerId containerId) throws IOException { - when(cgroup.memoryUsageInBytes(eq(containerId))).thenReturn(1228017664L); - when(cgroup.memoryLimitInBytes(eq(containerId))).thenReturn(2147483648L); - when(cgroup.memoryCacheInBytes(eq(containerId))).thenReturn(470790144L); + private void mockMemoryStats(ContainerId containerId) { + when(cgroup.resolveContainer(eq(containerId)).memory().readCurrent()).thenReturn(Size.from(1228017664L)); + when(cgroup.resolveContainer(eq(containerId)).memory().readMax()).thenReturn(Size.from(2147483648L)); + when(cgroup.resolveContainer(eq(containerId)).memory().readFileSystemCache()).thenReturn(Size.from(470790144L)); } private void mockCpuStats(ContainerId containerId) throws IOException { UnixPath proc = new UnixPath(fileSystem.getPath("/proc")); proc.createDirectories(); - when(cgroup.cpuStats(eq(containerId))).thenReturn(Map.of( + when(cgroup.resolveContainer(eq(containerId)).cpu().readStats()).thenReturn(Map.of( TOTAL_USAGE_USEC, 691675615472L, SYSTEM_USAGE_USEC, 262190000000L, USER_USAGE_USEC, 40900L, TOTAL_PERIODS, 1L, THROTTLED_PERIODS, 2L, THROTTLED_TIME_USEC, 3L)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java index 7676d0e1790..990b8c5997b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java @@ -5,9 +5,9 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.node.admin.cgroup.ControlGroup; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.container.CGroupV2; import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; @@ -60,7 +60,7 @@ public class ContainerTester implements AutoCloseable { private final ContainerEngineMock containerEngine = new ContainerEngineMock(); private final FileSystem fileSystem = TestFileSystem.create(); - final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, new CGroupV2(fileSystem), fileSystem)); + final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, mock(ControlGroup.class), fileSystem)); final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); final Orchestrator orchestrator = mock(Orchestrator.class); final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); |