diff options
author | Valerij Fredriksen <valerijf@yahooinc.com> | 2021-10-28 14:39:12 +0200 |
---|---|---|
committer | Valerij Fredriksen <valerijf@yahooinc.com> | 2021-10-28 14:39:12 +0200 |
commit | 2db9b00be157a7d4fdc29937cd0f3c5485c9128c (patch) | |
tree | 5b1f7f4fb409429f7cf8ccc463db76fc8b927e61 /node-admin | |
parent | 9a5d47e2098a595dc81c013930fb9fe8089bd9e1 (diff) |
Support CGroups v2
Diffstat (limited to 'node-admin')
13 files changed, 635 insertions, 248 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroup.java new file mode 100644 index 00000000000..d194198b2d7 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroup.java @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; + +import static com.yahoo.vespa.hosted.node.admin.container.ContainerStatsCollector.userHzToMicroSeconds; + +/** + * Read and write interface to the CGroup of a podman container. + * + * @author freva + */ +public interface CGroup { + + Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId); + OptionalInt cpuShares(ContainerId containerId); + + boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs); + boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares); + + Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException; + + /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ + long memoryLimitInBytes(ContainerId containerId) throws IOException; + + /** @return The total amount of memory currently being used by the cgroup and its descendants. */ + long memoryUsageInBytes(ContainerId containerId) throws IOException; + + /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */ + long memoryCacheInBytes(ContainerId containerId) throws IOException; + + enum CpuStatField { + TOTAL_USAGE_USEC(null/* in a dedicated file */, "usage_usec"), + USER_USAGE_USEC("user", "user_usec"), + SYSTEM_USAGE_USEC("system", "system_usec"), + TOTAL_PERIODS("nr_periods", "nr_periods"), + THROTTLED_PERIODS("nr_throttled", "nr_throttled"), + THROTTLED_TIME_USEC("throttled_time", "throttled_usec"); + + private final String v1Name; + private final String v2Name; + CpuStatField(String v1Name, String v2Name) { + this.v1Name = v1Name; + this.v2Name = v2Name; + } + + long parseValueV1(String value) { + long longValue = Long.parseLong(value); + switch (this) { + case THROTTLED_TIME_USEC: + case TOTAL_USAGE_USEC: + return longValue / 1000; // Value in ns + case USER_USAGE_USEC: + case SYSTEM_USAGE_USEC: + return userHzToMicroSeconds(longValue); + default: return longValue; + } + } + + long parseValueV2(String value) { + return Long.parseLong(value); + } + + static Optional<CpuStatField> fromV1Field(String name) { + return Arrays.stream(values()) + .filter(field -> name.equals(field.v1Name)) + .findFirst(); + } + + static Optional<CpuStatField> fromV2Field(String name) { + return Arrays.stream(values()) + .filter(field -> name.equals(field.v2Name)) + .findFirst(); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1.java new file mode 100644 index 00000000000..62e70ca26aa --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1.java @@ -0,0 +1,129 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.logging.Logger; +import java.util.stream.Stream; + +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.parseLong; + +/** + * Read and write interface to the CGroup V1 of a Podman container. + * + * @see <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/index.html">CGroups V1</a> + * @author freva + */ +public class CGroupV1 implements CGroup { + + private static final Logger logger = Logger.getLogger(CGroupV1.class.getName()); + + private final FileSystem fileSystem; + + public CGroupV1(FileSystem fileSystem) { + this.fileSystem = fileSystem; + } + + @Override + public Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId) { + OptionalInt quota = readCgroupsCpuInt(cfsQuotaPath(containerId)); + if (quota.isEmpty() || quota.getAsInt() < 0) return Optional.empty(); + OptionalInt period = readCgroupsCpuInt(cfsPeriodPath(containerId)); + if (period.isEmpty()) return Optional.empty(); + return Optional.of(new Pair<>(quota.getAsInt(), period.getAsInt())); + } + + @Override + public OptionalInt cpuShares(ContainerId containerId) { + return readCgroupsCpuInt(sharesPath(containerId)); + } + + @Override + public boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs) { + return writeCgroupsCpuInt(context, cfsQuotaPath(containerId), cpuQuotaUs) | + writeCgroupsCpuInt(context, cfsPeriodPath(containerId), periodUs); + } + + @Override + public boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares) { + return writeCgroupsCpuInt(context, sharesPath(containerId), shares); + } + + @Override + public Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException { + Map<CpuStatField, Long> stats = new HashMap<>(); + stats.put(CpuStatField.TOTAL_USAGE_USEC, parseLong(cpuacctPath(containerId).resolve("cpuacct.usage")) / 1000); + Stream.concat(Files.readAllLines(cpuacctPath(containerId).resolve("cpuacct.stat")).stream(), + Files.readAllLines(cpuacctPath(containerId).resolve("cpu.stat")).stream()) + .forEach(line -> { + String[] parts = line.split("\\s+"); + if (parts.length != 2) return; + CpuStatField.fromV1Field(parts[0]).ifPresent(field -> stats.put(field, field.parseValueV1(parts[1]))); + }); + return stats; + } + + @Override + public long memoryLimitInBytes(ContainerId containerId) throws IOException { + return parseLong(memoryPath(containerId).resolve("memory.limit_in_bytes")); + } + + @Override + public long memoryUsageInBytes(ContainerId containerId) throws IOException { + return parseLong(memoryPath(containerId).resolve("memory.usage_in_bytes")); + } + + @Override + public long memoryCacheInBytes(ContainerId containerId) throws IOException { + return parseLong(memoryPath(containerId).resolve("memory.stat"), "cache"); + } + + private Path cpuacctPath(ContainerId containerId) { + return fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-" + containerId + ".scope"); + } + + private Path cpuPath(ContainerId containerId) { + return fileSystem.getPath("/sys/fs/cgroup/cpu/machine.slice/libpod-" + containerId + ".scope"); + } + + private Path memoryPath(ContainerId containerId) { + return fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-" + containerId + ".scope"); + } + + private UnixPath cfsQuotaPath(ContainerId containerId) { + return new UnixPath(cpuPath(containerId).resolve("cpu.cfs_quota_us")); + } + + private UnixPath cfsPeriodPath(ContainerId containerId) { + return new UnixPath(cpuPath(containerId).resolve("cpu.cfs_period_us")); + } + + private UnixPath sharesPath(ContainerId containerId) { + return new UnixPath(cpuPath(containerId).resolve("cpu.shares")); + } + + private static OptionalInt readCgroupsCpuInt(UnixPath unixPath) { + return unixPath.readUtf8FileIfExists() + .map(s -> OptionalInt.of(Integer.parseInt(s.strip()))) + .orElseGet(OptionalInt::empty); + } + + private static boolean writeCgroupsCpuInt(NodeAgentContext context, UnixPath unixPath, int value) { + int currentValue = readCgroupsCpuInt(unixPath).orElseThrow(); + if (currentValue == value) return false; + + context.recordSystemModification(logger, "Updating " + unixPath + " from " + currentValue + " to " + value); + unixPath.writeUtf8File(Integer.toString(value)); + return true; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java new file mode 100644 index 00000000000..6b2d98a682a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java @@ -0,0 +1,133 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Read and write interface to the CGroup V1 of a Podman container. + * + * @see <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html">CGroups V2</a> + * @author freva + */ +public class CGroupV2 implements CGroup { + + private static final Logger logger = Logger.getLogger(CGroupV2.class.getName()); + private static final String MAX = "max"; + + private final FileSystem fileSystem; + + public CGroupV2(FileSystem fileSystem) { + this.fileSystem = fileSystem; + } + + @Override + public Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId) { + return cpuMaxPath(containerId).readUtf8FileIfExists() + .filter(s -> !s.startsWith(MAX)) + .map(s -> { + String[] parts = s.strip().split(" "); + return new Pair<>(Integer.parseInt(parts[0]), Integer.parseInt(parts[1])); + }); + } + + @Override + public OptionalInt cpuShares(ContainerId containerId) { + return cpuWeightPath(containerId).readUtf8FileIfExists() + .map(s -> OptionalInt.of(weightToShares(Integer.parseInt(s.strip())))) + .orElseGet(OptionalInt::empty); + } + + @Override + public boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs) { + String wanted = String.format("%s %d", cpuQuotaUs < 0 ? MAX : cpuQuotaUs, periodUs); + return writeCGroupsValue(context, cpuMaxPath(containerId), wanted); + } + + @Override + public boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares) { + return writeCGroupsValue(context, cpuWeightPath(containerId), Integer.toString(sharesToWeight(shares))); + } + + @Override + public Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException { + return Files.readAllLines(cgroupRoot(containerId).resolve("cpu.stat")).stream() + .map(line -> line.split("\\s+")) + .filter(parts -> parts.length == 2) + .flatMap(parts -> CpuStatField.fromV2Field(parts[0]).stream().map(field -> new Pair<>(field, field.parseValueV2(parts[1])))) + .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); + } + + @Override + public long memoryLimitInBytes(ContainerId containerId) throws IOException { + String limit = Files.readString(cgroupRoot(containerId).resolve("memory.max")).strip(); + return MAX.equals(limit) ? -1L : Long.parseLong(limit); + } + + @Override + public long memoryUsageInBytes(ContainerId containerId) throws IOException { + return parseLong(cgroupRoot(containerId).resolve("memory.current")); + } + + @Override + public long memoryCacheInBytes(ContainerId containerId) throws IOException { + return parseLong(cgroupRoot(containerId).resolve("memory.stat"), "file"); + } + + private Path cgroupRoot(ContainerId containerId) { + // crun path, runc path is without the 'container' directory + return fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-" + containerId + ".scope/container"); + } + + private UnixPath cpuMaxPath(ContainerId containerId) { + return new UnixPath(cgroupRoot(containerId).resolve("cpu.max")); + } + + private UnixPath cpuWeightPath(ContainerId containerId) { + return new UnixPath(cgroupRoot(containerId).resolve("cpu.weight")); + } + + private static boolean writeCGroupsValue(NodeAgentContext context, UnixPath unixPath, String value) { + String currentValue = unixPath.readUtf8File().strip(); + if (currentValue.equals(value)) return false; + + context.recordSystemModification(logger, "Updating " + unixPath + " from " + currentValue + " to " + value); + unixPath.writeUtf8File(value); + return true; + } + + // Must be same as in crun: https://github.com/containers/crun/blob/72c6e60ade0e4716fe2d8353f0d97d72cc8d1510/src/libcrun/cgroup.c#L3061 + static int sharesToWeight(int shares) { return (int) (1 + ((shares - 2L) * 9999) / 262142); } + static int weightToShares(int weight) { return (int) (2 + ((weight - 1L) * 262142) / 9999); } + + static long parseLong(Path path) throws IOException { + return Long.parseLong(Files.readString(path).trim()); + } + + static long parseLong(Path path, String fieldName) throws IOException { + return parseLong(Files.readAllLines(path), fieldName); + } + + static long parseLong(List<String> lines, String fieldName) { + for (String line : lines) { + String[] fields = line.split("\\s+"); + if (fields.length != 2) + throw new IllegalArgumentException("Expected line on the format 'key value', got: '" + line + "'"); + + if (fieldName.equals(fields[0])) return Long.parseLong(fields[1]); + } + throw new IllegalArgumentException("No such field: " + fieldName); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Cgroup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Cgroup.java deleted file mode 100644 index 1d87415b78e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Cgroup.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; - -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.util.OptionalInt; -import java.util.logging.Logger; - -/** - * Read and write interface to the v1 cgroup of a podman container. - * See <a href="https://man7.org/linux/man-pages/man7/cgroups.7.html">cgroups(7)</a> for background. - * - * @author hakon - */ -public class Cgroup { - - private static final Logger logger = Logger.getLogger(Cgroup.class.getName()); - - private final FileSystem fileSystem; - private final ContainerId containerId; - - public Cgroup(FileSystem fileSystem, ContainerId containerId) { - this.fileSystem = fileSystem; - this.containerId = containerId; - } - - public OptionalInt readCpuQuota() { - return readCgroupsCpuInt(cfsQuotaPath()); - } - - public OptionalInt readCpuPeriod() { - return readCgroupsCpuInt(cfsPeriodPath()); - } - - public OptionalInt readCpuShares() { - return readCgroupsCpuInt(sharesPath()); - } - - public boolean updateCpuQuota(NodeAgentContext context, int cpuQuotaUs) { - return writeCgroupsCpuInt(context, cfsQuotaPath(), cpuQuotaUs); - } - - public boolean updateCpuPeriod(NodeAgentContext context, int periodUs) { - return writeCgroupsCpuInt(context, cfsPeriodPath(), periodUs); - } - - public boolean updateCpuShares(NodeAgentContext context, int shares) { - return writeCgroupsCpuInt(context, sharesPath(), shares); - } - - /** Returns the path to the podman container's scope directory for the cpuacct controller. */ - public Path cpuacctPath() { - return fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-" + containerId + ".scope"); - } - - /** Returns the path to the podman container's scope directory for the cpu controller. */ - public Path cpuPath() { - return fileSystem.getPath("/sys/fs/cgroup/cpu/machine.slice/libpod-" + containerId + ".scope"); - } - - /** Returns the path to the podman container's scope directory for the memory controller. */ - public Path memoryPath() { - return fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-" + containerId + ".scope"); - } - - private UnixPath cfsQuotaPath() { - return new UnixPath(cpuPath().resolve("cpu.cfs_quota_us")); - } - - private UnixPath cfsPeriodPath() { - return new UnixPath(cpuPath().resolve("cpu.cfs_period_us")); - } - - private UnixPath sharesPath() { - return new UnixPath(cpuPath().resolve("cpu.shares")); - } - - private OptionalInt readCgroupsCpuInt(UnixPath unixPath) { - return unixPath.readUtf8FileIfExists() - .map(s -> OptionalInt.of(Integer.parseInt(s.strip()))) - .orElseGet(OptionalInt::empty); - } - - private boolean writeCgroupsCpuInt(NodeAgentContext context, UnixPath unixPath, int value) { - int currentValue = readCgroupsCpuInt(unixPath).orElseThrow(); - if (currentValue == value) { - return false; - } - - context.recordSystemModification(logger, "Updating " + unixPath + " from " + currentValue + " to " + value); - unixPath.writeUtf8File(Integer.toString(value)); - return true; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index fd38d38b381..af12a6201d3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -33,11 +33,11 @@ public class ContainerOperations { private final ContainerImagePruner imagePruner; private final ContainerStatsCollector containerStatsCollector; - public ContainerOperations(ContainerEngine containerEngine, FileSystem fileSystem) { + public ContainerOperations(ContainerEngine containerEngine, CGroup cgroup, FileSystem fileSystem) { this.containerEngine = Objects.requireNonNull(containerEngine); this.imageDownloader = new ContainerImageDownloader(containerEngine); this.imagePruner = new ContainerImagePruner(containerEngine, Clock.systemUTC()); - this.containerStatsCollector = new ContainerStatsCollector(Objects.requireNonNull(fileSystem)); + this.containerStatsCollector = new ContainerStatsCollector(cgroup, fileSystem); } public void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources) { @@ -86,7 +86,7 @@ public class ContainerOperations { } /** - * Suspend node and return output. Suspending a node means the node should be taken temporarly offline, + * Suspend node and return output. Suspending a node means the node should be taken temporarily offline, * such that maintenance of the node can be done (upgrading, rebooting, etc). */ public String suspendNode(NodeAgentContext context) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java index a7f6430035d..1838d8a8ac0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java @@ -87,7 +87,12 @@ public class ContainerResources { /** Returns true iff the CPU component(s) of between <code>this</code> and <code>other</code> are equal */ public boolean equalsCpu(ContainerResources other) { - return Math.abs(other.cpus - cpus) < 0.0001 && cpuShares == other.cpuShares; + return Math.abs(other.cpus - cpus) < 0.0001 && + // When using CGroups V2, CPU shares (range [2, 262144]) is mapped to CPU weight (range [1, 10000]), + // because there are ~26.2 shares/weight, we must allow for small deviation in cpuShares + // when comparing ContainerResources created from NodeResources vs one created from reading the + // CGroups weight file + Math.abs(cpuShares - other.cpuShares) < 28; } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java index fae4fc72145..168f319febd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java @@ -182,16 +182,16 @@ public class ContainerStats { public int getOnlineCpus() { return this.onlineCpus; } - /** Total CPU time (in ns) spent executing all the processes on this host */ + /** Total CPU time (in µs) spent executing all the processes on this host */ public long getSystemCpuUsage() { return this.systemCpuUsage; } - /** Total CPU time (in ns) spent running all the processes in this container */ + /** Total CPU time (in µs) spent running all the processes in this container */ public long getTotalUsage() { return totalUsage; } - /** Total CPU time (in ns) spent in kernel mode while executing processes in this container */ + /** Total CPU time (in µs) spent in kernel mode while executing processes in this container */ public long getUsageInKernelMode() { return usageInKernelMode; } - /** Total CPU time (in ns) processes in this container were throttled for */ + /** Total CPU time (in µs) processes in this container were throttled for */ public long getThrottledTime() { return throttledTime; } /** Number of periods with throttling enabled for this container */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java index e2176341bc0..67956892898 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java @@ -7,7 +7,6 @@ import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Objects; @@ -22,18 +21,25 @@ import java.util.Optional; */ class ContainerStatsCollector { + private final CGroup cgroup; private final FileSystem fileSystem; + private final int onlineCpus; - public ContainerStatsCollector(FileSystem fileSystem) { + ContainerStatsCollector(CGroup cgroup, FileSystem fileSystem) { + this(cgroup, fileSystem, Runtime.getRuntime().availableProcessors()); + } + + ContainerStatsCollector(CGroup cgroup, FileSystem fileSystem, int onlineCpus) { + this.cgroup = Objects.requireNonNull(cgroup); this.fileSystem = Objects.requireNonNull(fileSystem); + this.onlineCpus = onlineCpus; } /** Collect statistics for given container ID and PID */ public Optional<ContainerStats> collect(ContainerId containerId, int pid, String iface) { - Cgroup cgroup = new Cgroup(fileSystem, containerId); try { - ContainerStats.CpuStats cpuStats = collectCpuStats(cgroup); - ContainerStats.MemoryStats memoryStats = collectMemoryStats(cgroup); + ContainerStats.CpuStats cpuStats = collectCpuStats(containerId); + ContainerStats.MemoryStats memoryStats = collectMemoryStats(containerId); Map<String, ContainerStats.NetworkStats> networkStats = Map.of(iface, collectNetworkStats(iface, pid)); return Optional.of(new ContainerStats(networkStats, memoryStats, cpuStats)); } catch (NoSuchFileException ignored) { @@ -43,24 +49,21 @@ class ContainerStatsCollector { } } - private ContainerStats.CpuStats collectCpuStats(Cgroup cgroup) throws IOException { - List<String> cpuStatLines = Files.readAllLines(cpuStatPath(cgroup)); - long throttledActivePeriods = parseLong(cpuStatLines, "nr_periods"); - long throttledPeriods = parseLong(cpuStatLines, "nr_throttled"); - long throttledTime = parseLong(cpuStatLines, "throttled_time"); - return new ContainerStats.CpuStats(cpuCount(cgroup), - systemCpuUsage().toNanos(), - containerCpuUsage(cgroup).toNanos(), - containerCpuUsageSystem(cgroup).toNanos(), - throttledTime, - throttledActivePeriods, - throttledPeriods); + private ContainerStats.CpuStats collectCpuStats(ContainerId containerId) throws IOException { + Map<CGroup.CpuStatField, Long> cpuStats = cgroup.cpuStats(containerId); + return new ContainerStats.CpuStats(onlineCpus, + systemCpuUsage(), + cpuStats.get(CGroup.CpuStatField.TOTAL_USAGE_USEC), + cpuStats.get(CGroup.CpuStatField.SYSTEM_USAGE_USEC), + cpuStats.get(CGroup.CpuStatField.THROTTLED_TIME_USEC), + cpuStats.get(CGroup.CpuStatField.TOTAL_PERIODS), + cpuStats.get(CGroup.CpuStatField.THROTTLED_PERIODS)); } - private ContainerStats.MemoryStats collectMemoryStats(Cgroup cgroup) throws IOException { - long memoryLimitInBytes = parseLong(memoryLimitPath(cgroup)); - long memoryUsageInBytes = parseLong(memoryUsagePath(cgroup)); - long cachedInBytes = parseLong(memoryStatPath(cgroup), "cache"); + private ContainerStats.MemoryStats collectMemoryStats(ContainerId containerId) throws IOException { + long memoryLimitInBytes = cgroup.memoryLimitInBytes(containerId); + long memoryUsageInBytes = cgroup.memoryUsageInBytes(containerId); + long cachedInBytes = cgroup.memoryCacheInBytes(containerId); return new ContainerStats.MemoryStats(cachedInBytes, memoryUsageInBytes, memoryLimitInBytes); } @@ -82,34 +85,10 @@ class ContainerStatsCollector { throw new IllegalArgumentException("No statistics found for interface " + iface); } - /** Number of CPUs seen by given container */ - private int cpuCount(Cgroup cgroup) throws IOException { - return fields(Files.readString(perCpuUsagePath(cgroup))).length; - } - - /** Returns total CPU time spent executing all the processes on this host */ - private Duration systemCpuUsage() throws IOException { - long ticks = parseLong(fileSystem.getPath("/proc/stat"), "cpu"); - return ticksToDuration(ticks); - } - - /** Returns total CPU time spent running all processes inside given container */ - private Duration containerCpuUsage(Cgroup cgroup) throws IOException { - return Duration.ofNanos(parseLong(cpuUsagePath(cgroup))); - } - - /** Returns total CPU time spent in kernel/system mode while executing processes inside given container */ - private Duration containerCpuUsageSystem(Cgroup cgroup) throws IOException { - long ticks = parseLong(cpuacctStatPath(cgroup), "system"); - return ticksToDuration(ticks); - } - - private long parseLong(Path path) throws IOException { - return Long.parseLong(Files.readString(path).trim()); - } - - private long parseLong(Path path, String fieldName) throws IOException { - return parseLong(Files.readAllLines(path), fieldName); + /** Returns total CPU time in µs spent executing all the processes on this host */ + private long systemCpuUsage() throws IOException { + long ticks = parseLong(Files.readAllLines(fileSystem.getPath("/proc/stat")), "cpu"); + return userHzToMicroSeconds(ticks); } private long parseLong(List<String> lines, String fieldName) { @@ -129,38 +108,9 @@ class ContainerStatsCollector { return fileSystem.getPath("/proc/" + containerPid + "/net/dev"); } - private Path cpuacctStatPath(Cgroup cgroup) { - return cgroup.cpuacctPath().resolve("cpuacct.stat"); - } - - private Path cpuUsagePath(Cgroup cgroup) { - return cgroup.cpuacctPath().resolve("cpuacct.usage"); - } - - private Path perCpuUsagePath(Cgroup cgroup) { - return cgroup.cpuacctPath().resolve("cpuacct.usage_percpu"); - } - - private Path cpuStatPath(Cgroup cgroup) { - return cgroup.cpuacctPath().resolve("cpu.stat"); - } - - private Path memoryStatPath(Cgroup cgroup) { - return cgroup.memoryPath().resolve("memory.stat"); - } - - private Path memoryUsagePath(Cgroup cgroup) { - return cgroup.memoryPath().resolve("memory.usage_in_bytes"); - } - - private Path memoryLimitPath(Cgroup cgroup) { - return cgroup.memoryPath().resolve("memory.limit_in_bytes"); - } - - private static Duration ticksToDuration(long ticks) { + static long userHzToMicroSeconds(long ticks) { // Ideally we would read this from _SC_CLK_TCK, but then we need JNI. However, in practice this is always 100 on x86 Linux - long ticksPerSecond = 100; - return Duration.ofNanos((ticks * Duration.ofSeconds(1).toNanos()) / ticksPerSecond); + return ticks * 10_000; } private static String[] fields(String s) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1Test.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1Test.java new file mode 100644 index 00000000000..02ee68a245b --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1Test.java @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; + +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.USER_USAGE_USEC; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author freva + */ +public class CGroupV1Test { + + private static final ContainerId containerId = new ContainerId("4aec78cc"); + + private final FileSystem fileSystem = TestFileSystem.create(); + private final CGroup cgroup = new CGroupV1(fileSystem); + private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); + + @Test + public void updates_cpu_quota_and_period() { + assertEquals(Optional.empty(), cgroup.cpuQuotaPeriod(containerId)); + + UnixPath cpu = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpu/machine.slice/libpod-4aec78cc.scope")).createDirectories(); + cpu.resolve("cpu.cfs_period_us").writeUtf8File("123456\n"); + cpu.resolve("cpu.cfs_quota_us").writeUtf8File("-1\n"); + assertEquals(Optional.empty(), cgroup.cpuQuotaPeriod(containerId)); + + cpu.resolve("cpu.cfs_quota_us").writeUtf8File("456\n"); + assertEquals(Optional.of(new Pair<>(456, 123456)), cgroup.cpuQuotaPeriod(containerId)); + + assertFalse(cgroup.updateCpuQuotaPeriod(context, containerId, 456, 123456)); + + assertTrue(cgroup.updateCpuQuotaPeriod(context, containerId, 654, 123456)); + assertEquals(Optional.of(new Pair<>(654, 123456)), cgroup.cpuQuotaPeriod(containerId)); + } + + @Test + public void updates_cpu_shares() { + assertEquals(OptionalInt.empty(), cgroup.cpuShares(containerId)); + + UnixPath cpuPath = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpu/machine.slice/libpod-4aec78cc.scope")).createDirectories(); + cpuPath.resolve("cpu.shares").writeUtf8File("987\n"); + assertEquals(OptionalInt.of(987), cgroup.cpuShares(containerId)); + + assertFalse(cgroup.updateCpuShares(context, containerId, 987)); + + assertTrue(cgroup.updateCpuShares(context, containerId, 789)); + assertEquals(OptionalInt.of(789), cgroup.cpuShares(containerId)); + } + + @Test + public void reads_cpu_stats() throws IOException { + UnixPath cpuacctPath = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-4aec78cc.scope")).createDirectories(); + cpuacctPath.resolve("cpuacct.usage").writeUtf8File("91623711445\n"); + cpuacctPath.resolve("cpuacct.stat").writeUtf8File("user 7463\n" + + "system 1741\n"); + cpuacctPath.resolve("cpu.stat").writeUtf8File("nr_periods 2361\n" + + "nr_throttled 342\n" + + "throttled_time 131033468519\n"); + + assertEquals(Map.of(TOTAL_USAGE_USEC, 91623711L, SYSTEM_USAGE_USEC, 17410000L, USER_USAGE_USEC, 74630000L, + TOTAL_PERIODS, 2361L, THROTTLED_PERIODS, 342L, THROTTLED_TIME_USEC, 131033468L), cgroup.cpuStats(containerId)); + } + + @Test + public void reads_memory_metrics() throws IOException { + UnixPath memoryPath = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-4aec78cc.scope")).createDirectories(); + memoryPath.resolve("memory.usage_in_bytes").writeUtf8File("2525093888\n"); + assertEquals(2525093888L, cgroup.memoryUsageInBytes(containerId)); + + memoryPath.resolve("memory.limit_in_bytes").writeUtf8File("4322885632\n"); + assertEquals(4322885632L, cgroup.memoryLimitInBytes(containerId)); + + memoryPath.resolve("memory.stat").writeUtf8File("cache 296828928\n" + + "rss 2152587264\n" + + "rss_huge 1107296256\n" + + "shmem 135168\n" + + "mapped_file 270336\n"); + assertEquals(296828928L, cgroup.memoryCacheInBytes(containerId)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java new file mode 100644 index 00000000000..4bcd983133c --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java @@ -0,0 +1,118 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; + +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.USER_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.sharesToWeight; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.weightToShares; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author freva + */ +public class CGroupV2Test { + + private static final ContainerId containerId = new ContainerId("4aec78cc"); + + private final FileSystem fileSystem = TestFileSystem.create(); + private final CGroup cgroup = new CGroupV2(fileSystem); + private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); + private final UnixPath cgroupRoot = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-4aec78cc.scope/container")).createDirectories(); + + @Test + public void updates_cpu_quota_and_period() { + assertEquals(Optional.empty(), cgroup.cpuQuotaPeriod(containerId)); + + cgroupRoot.resolve("cpu.max").writeUtf8File("max 100000\n"); + assertEquals(Optional.empty(), cgroup.cpuQuotaPeriod(containerId)); + + cgroupRoot.resolve("cpu.max").writeUtf8File("456 123456\n"); + assertEquals(Optional.of(new Pair<>(456, 123456)), cgroup.cpuQuotaPeriod(containerId)); + + assertFalse(cgroup.updateCpuQuotaPeriod(context, containerId, 456, 123456)); + + assertTrue(cgroup.updateCpuQuotaPeriod(context, containerId, 654, 123456)); + assertEquals(Optional.of(new Pair<>(654, 123456)), cgroup.cpuQuotaPeriod(containerId)); + assertEquals("654 123456", cgroupRoot.resolve("cpu.max").readUtf8File()); + + assertTrue(cgroup.updateCpuQuotaPeriod(context, containerId, -1, 123456)); + assertEquals(Optional.empty(), cgroup.cpuQuotaPeriod(containerId)); + assertEquals("max 123456", cgroupRoot.resolve("cpu.max").readUtf8File()); + } + + @Test + public void updates_cpu_shares() { + assertEquals(OptionalInt.empty(), cgroup.cpuShares(containerId)); + + cgroupRoot.resolve("cpu.weight").writeUtf8File("1\n"); + assertEquals(OptionalInt.of(2), cgroup.cpuShares(containerId)); + + assertFalse(cgroup.updateCpuShares(context, containerId, 2)); + + assertTrue(cgroup.updateCpuShares(context, containerId, 12345)); + assertEquals(OptionalInt.of(12323), cgroup.cpuShares(containerId)); + } + + @Test + public void reads_cpu_stats() throws IOException { + cgroupRoot.resolve("cpu.stat").writeUtf8File("usage_usec 17794243\n" + + "user_usec 16099205\n" + + "system_usec 1695038\n" + + "nr_periods 12465\n" + + "nr_throttled 25\n" + + "throttled_usec 14256\n"); + + assertEquals(Map.of(TOTAL_USAGE_USEC, 17794243L, USER_USAGE_USEC, 16099205L, SYSTEM_USAGE_USEC, 1695038L, + TOTAL_PERIODS, 12465L, THROTTLED_PERIODS, 25L, THROTTLED_TIME_USEC, 14256L), cgroup.cpuStats(containerId)); + } + + @Test + public void reads_memory_metrics() throws IOException { + cgroupRoot.resolve("memory.current").writeUtf8File("2525093888\n"); + assertEquals(2525093888L, cgroup.memoryUsageInBytes(containerId)); + + cgroupRoot.resolve("memory.max").writeUtf8File("4322885632\n"); + assertEquals(4322885632L, cgroup.memoryLimitInBytes(containerId)); + + cgroupRoot.resolve("memory.stat").writeUtf8File("anon 3481600\n" + + "file 69206016\n" + + "kernel_stack 73728\n" + + "slab 3552304\n" + + "percpu 262336\n" + + "sock 73728\n" + + "shmem 8380416\n" + + "file_mapped 1081344\n" + + "file_dirty 135168\n"); + assertEquals(69206016L, cgroup.memoryCacheInBytes(containerId)); + } + + @Test + public void shares_to_weight_and_back_is_stable() { + for (int i = 2; i <= 262144; i++) { + int originalShares = i; // Must be effectively final to use in lambda :( + int roundTripShares = weightToShares(sharesToWeight(i)); + int diff = i - roundTripShares; + assertTrue(diff >= 0 && diff <= 27, // ~26.2 shares / weight + () -> "Original shares: " + originalShares + ", round trip shares: " + roundTripShares + ", diff: " + diff); + } + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java index e40265ac00d..e6c63220d35 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; +import java.nio.file.FileSystem; import java.util.List; import java.util.Map; import java.util.Set; @@ -22,7 +23,8 @@ public class ContainerOperationsTest { private final TestTaskContext context = new TestTaskContext(); private final ContainerEngineMock containerEngine = new ContainerEngineMock(); - private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, TestFileSystem.create()); + private final FileSystem fileSystem = TestFileSystem.create(); + private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, new CGroupV2(fileSystem), fileSystem); @Test public void no_managed_containers_running() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java index 82fb63b0036..7df5309961f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java @@ -5,12 +5,22 @@ import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; +import java.io.IOException; import java.nio.file.FileSystem; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.USER_USAGE_USEC; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * @author mpolden @@ -18,10 +28,11 @@ import static org.junit.Assert.assertTrue; public class ContainerStatsCollectorTest { private final FileSystem fileSystem = TestFileSystem.create(); + private final CGroup cgroup = mock(CGroup.class); @Test - public void collect() { - ContainerStatsCollector collector = new ContainerStatsCollector(fileSystem); + public void collect() throws IOException { + ContainerStatsCollector collector = new ContainerStatsCollector(cgroup, fileSystem, 24); ContainerId containerId = new ContainerId("id1"); int containerPid = 42; assertTrue("No stats found", collector.collect(containerId, containerPid, "eth0").isEmpty()); @@ -32,7 +43,7 @@ public class ContainerStatsCollectorTest { Optional<ContainerStats> stats = collector.collect(containerId, containerPid, "eth0"); assertTrue(stats.isPresent()); - assertEquals(new ContainerStats.CpuStats(24, 6049374780000000L, 691675615472L, + assertEquals(new ContainerStats.CpuStats(24, 6049374780000L, 691675615472L, 262190000000L, 3L, 1L, 2L), stats.get().getCpuStats()); assertEquals(new ContainerStats.MemoryStats(470790144L, 1228017664L, 2147483648L), @@ -50,68 +61,20 @@ public class ContainerStatsCollectorTest { " eth0: 22280813 118083 3 4 0 0 0 0 19859383 115415 5 6 0 0 0 0\n"); } - private void mockMemoryStats(ContainerId containerId) { - UnixPath root = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-" + containerId + ".scope")); - root.createDirectories(); - - root.resolve("memory.limit_in_bytes").writeUtf8File("2147483648\n"); - root.resolve("memory.usage_in_bytes").writeUtf8File("1228017664\n"); - root.resolve("memory.stat").writeUtf8File("cache 470790144\n" + - "rss 698699776\n" + - "rss_huge 526385152\n" + - "shmem 0\n" + - "mapped_file 811008\n" + - "dirty 405504\n" + - "writeback 0\n" + - "swap 0\n" + - "pgpgin 6938085\n" + - "pgpgout 6780573\n" + - "pgfault 14343186\n" + - "pgmajfault 0\n" + - "inactive_anon 0\n" + - "active_anon 699289600\n" + - "inactive_file 455516160\n" + - "active_file 13787136\n" + - "unevictable 0\n" + - "hierarchical_memory_limit 2147483648\n" + - "hierarchical_memsw_limit 4294967296\n" + - "total_cache 470790144\n" + - "total_rss 698699776\n" + - "total_rss_huge 526385152\n" + - "total_shmem 0\n" + - "total_mapped_file 811008\n" + - "total_dirty 405504\n" + - "total_writeback 0\n" + - "total_swap 0\n" + - "total_pgpgin 6938085\n" + - "total_pgpgout 6780573\n" + - "total_pgfault 14343186\n" + - "total_pgmajfault 0\n" + - "total_inactive_anon 0\n" + - "total_active_anon 699289600\n" + - "total_inactive_file 455516160\n" + - "total_active_file 13787136\n" + - "total_unevictable 0\n"); + private void mockMemoryStats(ContainerId containerId) throws IOException { + when(cgroup.memoryUsageInBytes(eq(containerId))).thenReturn(1228017664L); + when(cgroup.memoryLimitInBytes(eq(containerId))).thenReturn(2147483648L); + when(cgroup.memoryCacheInBytes(eq(containerId))).thenReturn(470790144L); } - private void mockCpuStats(ContainerId containerId) { - UnixPath root = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-" + containerId + ".scope")); + private void mockCpuStats(ContainerId containerId) throws IOException { UnixPath proc = new UnixPath(fileSystem.getPath("/proc")); - root.createDirectories(); proc.createDirectories(); - root.resolve("cpu.stat").writeUtf8File("nr_periods 1\n" + - "nr_throttled 2\n" + - "throttled_time 3\n"); - root.resolve("cpuacct.usage_percpu").writeUtf8File("25801608855 22529436415 25293652376 26212081533 " + - "27545883290 25357818592 33464821448 32568003867 " + - "28916742231 31771772292 34418037242 38417072233 " + - "26069101127 24568838237 23683334366 26824607997 " + - "24289870206 22249389818 32683986446 32444831154 " + - "30488394217 26840956322 31633747261 30838696584\n"); - root.resolve("cpuacct.usage").writeUtf8File("691675615472\n"); - root.resolve("cpuacct.stat").writeUtf8File("user 40900\n" + - "system 26219\n"); + when(cgroup.cpuStats(eq(containerId))).thenReturn(Map.of( + TOTAL_USAGE_USEC, 691675615472L, SYSTEM_USAGE_USEC, 262190000000L, USER_USAGE_USEC, 40900L, + TOTAL_PERIODS, 1L, THROTTLED_PERIODS, 2L, THROTTLED_TIME_USEC, 3L)); + proc.resolve("stat").writeUtf8File("cpu 7991366 978222 2346238 565556517 1935450 25514479 615206 0 0 0\n" + "cpu0 387906 61529 99088 23516506 42258 1063359 29882 0 0 0\n" + "cpu1 271253 49383 86149 23655234 41703 1061416 31885 0 0 0\n" + diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java index 4a26195dd3a..e2b53e71c4b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.container.CGroupV2; import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; @@ -54,7 +55,8 @@ public class ContainerTester implements AutoCloseable { private final Thread loopThread; private final ContainerEngineMock containerEngine = new ContainerEngineMock(); - final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, TestFileSystem.create())); + private final FileSystem fileSystem = TestFileSystem.create(); + final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, new CGroupV2(fileSystem), fileSystem)); final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); final Orchestrator orchestrator = mock(Orchestrator.class); final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); |