From 4bb82cc75fe70611b2a8c7c3d16165e1d85355e4 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Thu, 12 May 2022 11:40:36 +0200 Subject: Add CGroupResourceLimits class. --- vespalib/CMakeLists.txt | 2 + vespalib/src/apps/vespa-resource-limits/.gitignore | 1 + .../src/apps/vespa-resource-limits/CMakeLists.txt | 9 + .../apps/vespa-resource-limits/resource_limits.cpp | 13 ++ .../util/cgroup_resource_limits/CMakeLists.txt | 9 + .../cgroup_resource_limits_test.cpp | 75 +++++++ .../cgroup/cpu/cpu.cfs_period_us | 1 + .../cgroup/cpu/cpu.cfs_quota_us | 1 + .../cgroup/memory/memory.limit_in_bytes | 1 + .../cgroup_v1_container/self | 2 + .../cgroup/cpu/group2/cpu.cfs_period_us | 1 + .../cgroup/cpu/group2/cpu.cfs_quota_us | 1 + .../cgroup/memory/group1/memory.limit_in_bytes | 1 + .../cgroup_resource_limits/cgroup_v1_host/self | 2 + .../cgroup/cpu/group2/cpu.cfs_period_us | 1 + .../cgroup/cpu/group2/cpu.cfs_quota_us | 1 + .../cgroup/cpu/group2/group4/cpu.cfs_period_us | 1 + .../cgroup/cpu/group2/group4/cpu.cfs_quota_us | 1 + .../memory/group1/group3/memory.limit_in_bytes | 1 + .../cgroup/memory/group1/memory.limit_in_bytes | 1 + .../cgroup_v1_host_nested/self | 2 + .../cgroup/cpu/group2/cpu.cfs_period_us | 1 + .../cgroup/cpu/group2/cpu.cfs_quota_us | 1 + .../cgroup/memory/group1/memory.limit_in_bytes | 1 + .../cgroup_v1_host_no_limit/self | 2 + .../cgroup_v2_container/cgroup/cpu.max | 1 + .../cgroup_v2_container/cgroup/memory.max | 1 + .../cgroup_v2_container/self | 1 + .../cgroup_v2_host/cgroup/group1/cpu.max | 1 + .../cgroup_v2_host/cgroup/group1/memory.max | 1 + .../cgroup_resource_limits/cgroup_v2_host/self | 1 + .../cgroup_v2_host_nested/cgroup/group1/cpu.max | 1 + .../cgroup/group1/group2/cpu.max | 1 + .../cgroup/group1/group2/memory.max | 1 + .../cgroup_v2_host_nested/cgroup/group1/memory.max | 1 + .../cgroup_v2_host_nested/self | 1 + .../cgroup_v2_host_no_limit/cgroup/group1/cpu.max | 1 + .../cgroup/group1/memory.max | 1 + .../cgroup_v2_host_no_limit/self | 1 + vespalib/src/vespa/vespalib/util/CMakeLists.txt | 2 + .../vespa/vespalib/util/cgroup_resource_limits.cpp | 235 +++++++++++++++++++++ .../vespa/vespalib/util/cgroup_resource_limits.h | 47 +++++ .../src/vespa/vespalib/util/resource_limits.cpp | 33 +++ vespalib/src/vespa/vespalib/util/resource_limits.h | 23 ++ 44 files changed, 486 insertions(+) create mode 100644 vespalib/src/apps/vespa-resource-limits/.gitignore create mode 100644 vespalib/src/apps/vespa-resource-limits/CMakeLists.txt create mode 100644 vespalib/src/apps/vespa-resource-limits/resource_limits.cpp create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/CMakeLists.txt create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_period_us create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_quota_us create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/memory/memory.limit_in_bytes create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/self create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_period_us create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_quota_us create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/memory/group1/memory.limit_in_bytes create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/self create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_period_us create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_quota_us create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_period_us create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_quota_us create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/group3/memory.limit_in_bytes create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/memory.limit_in_bytes create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/self create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_period_us create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_quota_us create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/memory/group1/memory.limit_in_bytes create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/self create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/cpu.max create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/memory.max create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/self create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/cpu.max create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/memory.max create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/self create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/cpu.max create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/cpu.max create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/memory.max create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/memory.max create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/self create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/cpu.max create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/memory.max create mode 100644 vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/self create mode 100644 vespalib/src/vespa/vespalib/util/cgroup_resource_limits.cpp create mode 100644 vespalib/src/vespa/vespalib/util/cgroup_resource_limits.h create mode 100644 vespalib/src/vespa/vespalib/util/resource_limits.cpp create mode 100644 vespalib/src/vespa/vespalib/util/resource_limits.h (limited to 'vespalib') diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 7796ce7df28..0ac193ea694 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_define_module( src/apps/make_fixture_macros src/apps/vespa-detect-hostname src/apps/vespa-drop-file-from-cache + src/apps/vespa-resource-limits src/apps/vespa-tsan-digest src/apps/vespa-validate-hostname @@ -146,6 +147,7 @@ vespa_define_module( src/tests/typify src/tests/unwind_message src/tests/util/bfloat16 + src/tests/util/cgroup_resource_limits src/tests/util/file_area_freelist src/tests/util/generationhandler src/tests/util/generationhandler_stress diff --git a/vespalib/src/apps/vespa-resource-limits/.gitignore b/vespalib/src/apps/vespa-resource-limits/.gitignore new file mode 100644 index 00000000000..58a2938f452 --- /dev/null +++ b/vespalib/src/apps/vespa-resource-limits/.gitignore @@ -0,0 +1 @@ +/vespa-resource-limits diff --git a/vespalib/src/apps/vespa-resource-limits/CMakeLists.txt b/vespalib/src/apps/vespa-resource-limits/CMakeLists.txt new file mode 100644 index 00000000000..566b23058cf --- /dev/null +++ b/vespalib/src/apps/vespa-resource-limits/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_vespa-resource-limits_app + SOURCES + resource_limits.cpp + OUTPUT_NAME vespa-resource-limits + INSTALL bin + DEPENDS + vespalib +) diff --git a/vespalib/src/apps/vespa-resource-limits/resource_limits.cpp b/vespalib/src/apps/vespa-resource-limits/resource_limits.cpp new file mode 100644 index 00000000000..59bea7d0fd8 --- /dev/null +++ b/vespalib/src/apps/vespa-resource-limits/resource_limits.cpp @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include + +int +main(int, char**) +{ + auto limits = vespalib::ResourceLimits::create(); + std::cout << "Memory limit: " << limits.memory() << '\n' << + "Cpu limit: " << limits.cpu() << std::endl; + return 0; +} diff --git a/vespalib/src/tests/util/cgroup_resource_limits/CMakeLists.txt b/vespalib/src/tests/util/cgroup_resource_limits/CMakeLists.txt new file mode 100644 index 00000000000..79414f9ef9f --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_cgroup_resource_limits_test_app TEST + SOURCES + cgroup_resource_limits_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_cgroup_resource_limits_test_app COMMAND vespalib_cgroup_resource_limits_test_app) diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp new file mode 100644 index 00000000000..a4e57f69ab8 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include + +namespace vespalib { + +class CGroupResourceLimitsTest : public ::testing::Test +{ +protected: + CGroupResourceLimitsTest(); + ~CGroupResourceLimitsTest(); + void check_limits(const std::string &name, const std::optional& memory_limit, const std::optional& cpu_limit); +}; + +CGroupResourceLimitsTest::CGroupResourceLimitsTest() = default; +CGroupResourceLimitsTest::~CGroupResourceLimitsTest() = default; + +void +CGroupResourceLimitsTest::check_limits(const std::string &base, const std::optional& memory_limit, const std::optional& cpu_limit) +{ + CGroupResourceLimits cg_limits(base + "/cgroup", base + "/self"); + EXPECT_EQ(memory_limit, cg_limits.get_memory_limit()); + EXPECT_EQ(cpu_limit, cg_limits.get_cpu_limit()); +} + +TEST_F(CGroupResourceLimitsTest, no_cgroup) +{ + check_limits("no_cgroup", std::nullopt, std::nullopt); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v1_host) +{ + check_limits("cgroup_v1_host", 4_Mi, 3); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v1_host_nested) +{ + check_limits("cgroup_v1_host_nested", 5_Mi, 4); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v1_host_no_limit) +{ + check_limits("cgroup_v1_host_no_limit", std::nullopt, std::nullopt); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v1_container) +{ + check_limits("cgroup_v1_container", 8_Mi, 5); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v2_host) +{ + check_limits("cgroup_v2_host", 12_Mi, 7); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v2_host_nested) +{ + check_limits("cgroup_v2_host_nested", 13_Mi, 8); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v2_host_no_limit) +{ + check_limits("cgroup_v2_host_no_limit", std::nullopt, std::nullopt); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v2_container) +{ + check_limits("cgroup_v2_container", 16_Mi, 9); +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_period_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_period_us new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_quota_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_quota_us new file mode 100644 index 00000000000..354b2529b29 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_quota_us @@ -0,0 +1 @@ +500000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/memory/memory.limit_in_bytes b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/memory/memory.limit_in_bytes new file mode 100644 index 00000000000..16edb4cc19a --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/memory/memory.limit_in_bytes @@ -0,0 +1 @@ +8388608 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/self new file mode 100644 index 00000000000..ee80ea30458 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/self @@ -0,0 +1,2 @@ +1:memory:/group1 +1:cpu,cpuacct:/group2 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_period_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_period_us new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_quota_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_quota_us new file mode 100644 index 00000000000..67f9d558228 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_quota_us @@ -0,0 +1 @@ +300000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/memory/group1/memory.limit_in_bytes b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/memory/group1/memory.limit_in_bytes new file mode 100644 index 00000000000..3f7803daddf --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/memory/group1/memory.limit_in_bytes @@ -0,0 +1 @@ +4194304 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/self new file mode 100644 index 00000000000..ee80ea30458 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/self @@ -0,0 +1,2 @@ +1:memory:/group1 +1:cpu,cpuacct:/group2 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_period_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_period_us new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_quota_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_quota_us new file mode 100644 index 00000000000..004d15285e7 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_quota_us @@ -0,0 +1 @@ +400000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_period_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_period_us new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_quota_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_quota_us new file mode 100644 index 00000000000..354b2529b29 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_quota_us @@ -0,0 +1 @@ +500000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/group3/memory.limit_in_bytes b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/group3/memory.limit_in_bytes new file mode 100644 index 00000000000..faefc1fbffc --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/group3/memory.limit_in_bytes @@ -0,0 +1 @@ +6291456 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/memory.limit_in_bytes b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/memory.limit_in_bytes new file mode 100644 index 00000000000..062933f0941 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/memory.limit_in_bytes @@ -0,0 +1 @@ +5242880 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/self new file mode 100644 index 00000000000..98a1f55ba89 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/self @@ -0,0 +1,2 @@ +1:memory:/group1/group3 +1:cpu,cpuacct:/group2/group4 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_period_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_period_us new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_quota_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_quota_us new file mode 100644 index 00000000000..3a2e3f4984a --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_quota_us @@ -0,0 +1 @@ +-1 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/memory/group1/memory.limit_in_bytes b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/memory/group1/memory.limit_in_bytes new file mode 100644 index 00000000000..564113cfaff --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/memory/group1/memory.limit_in_bytes @@ -0,0 +1 @@ +9223372036854771712 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/self new file mode 100644 index 00000000000..ee80ea30458 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/self @@ -0,0 +1,2 @@ +1:memory:/group1 +1:cpu,cpuacct:/group2 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/cpu.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/cpu.max new file mode 100644 index 00000000000..06f6446c456 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/cpu.max @@ -0,0 +1 @@ +900000 100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/memory.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/memory.max new file mode 100644 index 00000000000..27f897b99ff --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/memory.max @@ -0,0 +1 @@ +16777216 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/self new file mode 100644 index 00000000000..f85fa774795 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/self @@ -0,0 +1 @@ +0::/group1 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/cpu.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/cpu.max new file mode 100644 index 00000000000..fd6b2b4e326 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/cpu.max @@ -0,0 +1 @@ +700000 100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/memory.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/memory.max new file mode 100644 index 00000000000..372ed62fb4b --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/memory.max @@ -0,0 +1 @@ +12582912 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/self new file mode 100644 index 00000000000..f85fa774795 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/self @@ -0,0 +1 @@ +0::/group1 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/cpu.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/cpu.max new file mode 100644 index 00000000000..5c5f99b398b --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/cpu.max @@ -0,0 +1 @@ +800000 100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/cpu.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/cpu.max new file mode 100644 index 00000000000..06f6446c456 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/cpu.max @@ -0,0 +1 @@ +900000 100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/memory.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/memory.max new file mode 100644 index 00000000000..9812e3d8f08 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/memory.max @@ -0,0 +1 @@ +14680064 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/memory.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/memory.max new file mode 100644 index 00000000000..03b7c4afd21 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/memory.max @@ -0,0 +1 @@ +13631488 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/self new file mode 100644 index 00000000000..b2c3ecd563b --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/self @@ -0,0 +1 @@ +0::/group1/group2 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/cpu.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/cpu.max new file mode 100644 index 00000000000..1c1d3e7c303 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/cpu.max @@ -0,0 +1 @@ +max 100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/memory.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/memory.max new file mode 100644 index 00000000000..355295a05af --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/memory.max @@ -0,0 +1 @@ +max diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/self new file mode 100644 index 00000000000..f85fa774795 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/self @@ -0,0 +1 @@ +0::/group1 diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index f7bdd4427e3..70b8347acc2 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT binary_hamming_distance.cpp blockingthreadstackexecutor.cpp box.cpp + cgroup_resource_limits.cpp classname.cpp compress.cpp compressor.cpp @@ -50,6 +51,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT rcuvector.cpp regexp.cpp require.cpp + resource_limits.cpp reusable_set.cpp reusable_set_handle.cpp reusable_set_pool.cpp diff --git a/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.cpp b/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.cpp new file mode 100644 index 00000000000..35094c8aeaa --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.cpp @@ -0,0 +1,235 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "cgroup_resource_limits.h" +#include "round_up_to_page_size.h" +#include +#include +#include +#include +#include +#include + + +namespace vespalib { + +namespace { + +std::string +combine_paths(const std::string& base_path, const std::string& controller, const std::string &cgroup_path) +{ + std::ostringstream os; + os << base_path; + if (!controller.empty()) { + os << "/" << controller; + } + if (!cgroup_path.empty() && cgroup_path != "/") { + if (cgroup_path[0] != '/') { + os << "/"; + } + os << cgroup_path; + } + return os.str(); +} + +const std::string empty_std_string; + +} + +CGroupResourceLimits::CGroupResourceLimits() + : CGroupResourceLimits("/sys/fs/cgroup", "/proc/self/cgroup") +{ +} + +CGroupResourceLimits::CGroupResourceLimits(const std::string& base_path, const std::string& map_path) + : _memory_limit(), + _cpu_limit(), + _base_path(base_path), + _map_path(map_path), + _cgroup_v2_path(), + _cgroup_v1_paths() +{ + get_cgroup_paths(); + if (!_cgroup_v1_paths.empty()) { + get_limits_v1(); + } else { + get_limits_v2(); + } +} + +CGroupResourceLimits::~CGroupResourceLimits() = default; + +void +CGroupResourceLimits::apply_memory_limit(uint64_t memory_limit) +{ + if (!_memory_limit.has_value() || _memory_limit.value() > memory_limit) { + _memory_limit = memory_limit; + } +} + +void +CGroupResourceLimits::apply_cpu_limit(uint32_t cpu_limit) +{ + if (!_cpu_limit.has_value() || _cpu_limit.value() > cpu_limit) { + _cpu_limit = cpu_limit; + } +} + +void +CGroupResourceLimits::get_cgroup_paths() +{ + /* + * See manual page for cgroups(7) on linux for format of /proc/[pid]/cgroup. + */ + std::ifstream cg_self(_map_path); + std::string line; + while (std::getline(cg_self, line)) { + if (line.size() >= 3 && line.substr(0, 3) == "0::") { + _cgroup_v2_path = line.substr(3); + continue; + } + auto col_pos = line.find(':'); + if (col_pos != std::string::npos) { + auto name_pos = col_pos + 1; + col_pos = line.find(':', name_pos); + if (col_pos != std::string::npos) { + auto cg_path = line.substr(col_pos + 1); + auto comma_pos = line.find(',', name_pos); + while (comma_pos < col_pos) { + auto subsystem = line.substr(name_pos, comma_pos - name_pos); + _cgroup_v1_paths[subsystem] = cg_path; + name_pos = comma_pos + 1; + comma_pos = line.find(',', name_pos); + } + auto subsystem = line.substr(name_pos, col_pos - name_pos); + _cgroup_v1_paths[subsystem] = cg_path; + } + } + } +} + +void +CGroupResourceLimits::foreach_cgroup_level(const std::string& controller, const std::string& cgroup_path, const std::function& callback) +{ + auto dir = combine_paths(_base_path, controller, empty_std_string); + if (!isDirectory(dir)) { + return; + } + callback(dir); + if (cgroup_path.empty() || cgroup_path == "/") { + return; + } + auto slash_pos = cgroup_path.find('/', 1); + while (slash_pos != std::string::npos) { + dir = combine_paths(_base_path, controller, cgroup_path.substr(0, slash_pos)); + if (!isDirectory(dir)) { + return; + } + callback(dir); + slash_pos = cgroup_path.find('/', slash_pos + 1); + } + dir = combine_paths(_base_path, controller, cgroup_path); + if (isDirectory(dir)) { + callback(dir); + } +} + +void +CGroupResourceLimits::foreach_cgroup_v1_level(const std::string& controller, const std::function& callback) +{ + auto itr = _cgroup_v1_paths.find(controller); + if (itr != _cgroup_v1_paths.end()) { + foreach_cgroup_level(controller, itr->second, callback); + } else { + foreach_cgroup_level(controller, empty_std_string, callback); + } +} + +void +CGroupResourceLimits::foreach_cgroup_v2_level(const std::function& callback) +{ + foreach_cgroup_level(empty_std_string, _cgroup_v2_path, callback); +} + +void +CGroupResourceLimits::get_memory_limits_v1(const std::string& dir) +{ + std::ifstream limitfile(dir + "/memory.limit_in_bytes"); + uint64_t memory_limit = std::numeric_limits::max(); + limitfile >> memory_limit; + if (limitfile.good() && (memory_limit < std::numeric_limits::max() - (round_up_to_page_size(1) - 1))) { + apply_memory_limit(memory_limit); + } +} + +void +CGroupResourceLimits::get_memory_limits_v1() +{ + foreach_cgroup_v1_level("memory", [this](const std::string& dir) { get_memory_limits_v1(dir); }); +} + +void +CGroupResourceLimits::get_cpu_limits_v1(const std::string& dir) +{ + int32_t cpu_cfs_period_us = 0; + int32_t cpu_cfs_quota_us = 0; + std::ifstream period_file(dir + "/cpu.cfs_period_us"); + std::ifstream quota_file(dir + "/cpu.cfs_quota_us"); + period_file >> cpu_cfs_period_us; + quota_file >> cpu_cfs_quota_us; + if (quota_file.good() && period_file.good() && cpu_cfs_quota_us >= 0 && cpu_cfs_period_us > 0) { + auto cpu = std::max(1.0, std::ceil(static_cast(cpu_cfs_quota_us) / cpu_cfs_period_us)); + apply_cpu_limit(cpu); + } +} + +void +CGroupResourceLimits::get_cpu_limits_v1() +{ + foreach_cgroup_v1_level("cpu", [this](const std::string& dir) { get_cpu_limits_v1(dir); }); +} + +void +CGroupResourceLimits::get_limits_v1() +{ + get_memory_limits_v1(); + get_cpu_limits_v1(); +} + +void +CGroupResourceLimits::get_memory_limits_v2(const std::string& dir) +{ + std::ifstream limitfile(dir + "/memory.max"); + uint64_t memory_limit = std::numeric_limits::max(); + limitfile >> memory_limit; + if (limitfile.good()) { + apply_memory_limit(memory_limit); + } +} + +void +CGroupResourceLimits::get_cpu_limits_v2(const std::string& dir) +{ + int32_t cpu_cfs_period_us = 0; + int32_t cpu_cfs_quota_us = 0; + std::ifstream cpu_max_file(dir + "/cpu.max"); + cpu_max_file >> cpu_cfs_quota_us >> cpu_cfs_period_us; + if (cpu_max_file.good() && cpu_cfs_quota_us >= 0 && cpu_cfs_period_us > 0) { + auto cpu = std::max(1.0, std::ceil(static_cast(cpu_cfs_quota_us) / cpu_cfs_period_us)); + apply_cpu_limit(cpu); + } +} + +void +CGroupResourceLimits::get_limits_v2(const std::string& dir) +{ + get_memory_limits_v2(dir); + get_cpu_limits_v2(dir); +} + +void +CGroupResourceLimits::get_limits_v2() +{ + foreach_cgroup_v2_level([this](const std::string& dir) { get_limits_v2(dir); }); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.h b/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.h new file mode 100644 index 00000000000..04fc0384c1a --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.h @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include +#include +#include +#include + +namespace vespalib { + +/* + * Class for getting cgroup resource limits. + */ +class CGroupResourceLimits { + std::optional _memory_limit; + std::optional _cpu_limit; + std::string _base_path; // "/sys/fs/cgroup" + std::string _map_path; // "/proc/self/cgroup" + std::string _cgroup_v2_path; // Same cgroup path for all controllers + std::map _cgroup_v1_paths; // controller -> cgroup path + + void apply_memory_limit(uint64_t memory_limit); + void apply_cpu_limit(uint32_t cpu_limit); + void get_cgroup_paths(); + void foreach_cgroup_level(const std::string& controller, const std::string& cgroup_path, const std::function& callback); + void foreach_cgroup_v1_level(const std::string& controller, const std::function& callback); + void foreach_cgroup_v2_level(const std::function& callback); + void get_memory_limits_v1(const std::string& dir); + void get_memory_limits_v1(); + void get_cpu_limits_v1(const std::string& dir); + void get_cpu_limits_v1(); + void get_limits_v1(); + void get_memory_limits_v2(const std::string& dir); + void get_cpu_limits_v2(const std::string& dir); + void get_limits_v2(const std::string& dir); + void get_limits_v2(); +public: + CGroupResourceLimits(); + CGroupResourceLimits(const std::string& base_path, const std::string& map_path); + ~CGroupResourceLimits(); + const std::optional& get_memory_limit() const noexcept { return _memory_limit; } + const std::optional& get_cpu_limit() const noexcept { return _cpu_limit; } +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/resource_limits.cpp b/vespalib/src/vespa/vespalib/util/resource_limits.cpp new file mode 100644 index 00000000000..85c479a67b6 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/resource_limits.cpp @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "resource_limits.h" +#include "cgroup_resource_limits.h" +#include +#include + +namespace vespalib { + +ResourceLimits::ResourceLimits(uint64_t memory, uint32_t cpu) + : _memory(memory), + _cpu(cpu) +{ +} + +ResourceLimits +ResourceLimits::create() +{ + uint64_t memory = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); + uint32_t cpu = std::thread::hardware_concurrency(); + CGroupResourceLimits cgroup_limits; + auto& cg_memory = cgroup_limits.get_memory_limit(); + auto& cg_cpu = cgroup_limits.get_cpu_limit(); + if (cg_memory.has_value() && cg_memory.value() < memory) { + memory = cg_memory.value(); + } + if (cg_cpu.has_value() && cg_cpu.value() < cpu) { + cpu = cg_cpu.value(); + } + return ResourceLimits(memory, cpu); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/resource_limits.h b/vespalib/src/vespa/vespalib/util/resource_limits.h new file mode 100644 index 00000000000..7255cef9df5 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/resource_limits.h @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include + +namespace vespalib { + +/* + * Class for getting resource limits. + */ +class ResourceLimits { + uint64_t _memory; + uint32_t _cpu; + + ResourceLimits(uint64_t memory, uint32_t cpu); +public: + static ResourceLimits create(); + uint64_t memory() const noexcept { return _memory; } + uint32_t cpu() const noexcept { return _cpu; } +}; + +} -- cgit v1.2.3