diff options
author | Geir Storli <geirst@oath.com> | 2017-12-01 00:44:12 +0000 |
---|---|---|
committer | Geir Storli <geirst@oath.com> | 2017-12-01 00:44:12 +0000 |
commit | 6255d75202bad5b1e783033e7736873ed7cbd7b0 (patch) | |
tree | 217a85b802f543ae92189fec8b03fb4e970cb136 /searchcore | |
parent | f27aaffbd70a2cf3824a8d3dbf45569dc833293e (diff) |
Add metrics on how far away from disk and memory resource limits proton is.
Diffstat (limited to 'searchcore')
8 files changed, 59 insertions, 46 deletions
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp index 4dd8c43d9b2..fb0c9b1f3a9 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp @@ -3,14 +3,12 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/server/disk_mem_usage_filter.h> +#include <vespa/searchcore/proton/server/resource_usage_state.h> -using proton::DiskMemUsageFilter; -using proton::HwInfo; +using namespace proton; namespace fs = std::experimental::filesystem; -namespace { - struct Fixture { DiskMemUsageFilter _filter; @@ -18,13 +16,13 @@ struct Fixture using Config = DiskMemUsageFilter::Config; Fixture() - : _filter(HwInfo(HwInfo::Disk(100, false, false), HwInfo::Memory(64 * 1024 * 1024), HwInfo::Cpu(0))) + : _filter(HwInfo(HwInfo::Disk(100, false, false), HwInfo::Memory(1000), HwInfo::Cpu(0))) { - _filter.setDiskUsedSize(0); - _filter.setMemoryStats(vespalib::ProcessMemoryStats(10000000, - 10000001, - 10000002, - 10000003, + _filter.setDiskUsedSize(20); + _filter.setMemoryStats(vespalib::ProcessMemoryStats(297, + 298, + 299, + 300, 42)); } @@ -48,16 +46,14 @@ struct Fixture void triggerMemoryLimit() { - _filter.setMemoryStats(vespalib::ProcessMemoryStats(58720259, - 58720258, - 58720257, - 58720256, + _filter.setMemoryStats(vespalib::ProcessMemoryStats(897, + 898, + 899, + 900, 43)); } }; -} - TEST_F("Check that default filter allows write", Fixture) { f.testWrite(""); @@ -70,28 +66,40 @@ TEST_F("Check that stats are wired through", Fixture) EXPECT_EQUAL(43u, f._filter.getMemoryStats().getMappingsCount()); } +void +assertResourceUsage(double usage, double limit, double utilization, const ResourceUsageState &state) +{ + EXPECT_EQUAL(usage, state.usage()); + EXPECT_EQUAL(limit, state.limit()); + EXPECT_EQUAL(utilization, state.utilization()); +} + TEST_F("Check that disk limit can be reached", Fixture) { f._filter.setConfig(Fixture::Config(1.0, 0.8)); + TEST_DO(assertResourceUsage(0.2, 0.8, 0.25, f._filter.usageState().diskState())); f.triggerDiskLimit(); f.testWrite("diskLimitReached: { " "action: \"add more content nodes\", " "reason: \"disk used (0.9) > disk limit (0.8)\", " "stats: { " "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); + TEST_DO(assertResourceUsage(0.9, 0.8, 1.125, f._filter.usageState().diskState())); } TEST_F("Check that memory limit can be reached", Fixture) { f._filter.setConfig(Fixture::Config(0.8, 1.0)); + TEST_DO(assertResourceUsage(0.3, 0.8, 0.375, f._filter.usageState().memoryState())); f.triggerMemoryLimit(); f.testWrite("memoryLimitReached: { " "action: \"add more content nodes\", " - "reason: \"memory used (0.875) > memory limit (0.8)\", " + "reason: \"memory used (0.9) > memory limit (0.8)\", " "stats: { " - "mapped: { virt: 58720259, rss: 58720258}, " - "anonymous: { virt: 58720257, rss: 58720256}, " - "physicalMemory: 67108864, memoryUsed: 0.875, memoryLimit: 0.8}}"); + "mapped: { virt: 897, rss: 898}, " + "anonymous: { virt: 899, rss: 900}, " + "physicalMemory: 1000, memoryUsed: 0.9, memoryLimit: 0.8}}"); + TEST_DO(assertResourceUsage(0.9, 0.8, 1.125, f._filter.usageState().memoryState())); } TEST_F("Check that both disk limit and memory limit can be reached", Fixture) @@ -101,11 +109,11 @@ TEST_F("Check that both disk limit and memory limit can be reached", Fixture) f.triggerDiskLimit(); f.testWrite("memoryLimitReached: { " "action: \"add more content nodes\", " - "reason: \"memory used (0.875) > memory limit (0.8)\", " + "reason: \"memory used (0.9) > memory limit (0.8)\", " "stats: { " - "mapped: { virt: 58720259, rss: 58720258}, " - "anonymous: { virt: 58720257, rss: 58720256}, " - "physicalMemory: 67108864, memoryUsed: 0.875, memoryLimit: 0.8}}, " + "mapped: { virt: 897, rss: 898}, " + "anonymous: { virt: 899, rss: 900}, " + "physicalMemory: 1000, memoryUsed: 0.9, memoryLimit: 0.8}}, " "diskLimitReached: { " "action: \"add more content nodes\", " "reason: \"disk used (0.9) > disk limit (0.8)\", " diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp index a699cf491e2..d944287243a 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp @@ -7,7 +7,9 @@ namespace proton { ResourceUsageMetrics::ResourceUsageMetrics(metrics::MetricSet *parent) : MetricSet("resource_usage", "", "Usage metrics for various resources in this search engine", parent), disk("disk", "", "The relative amount of disk space used on this machine (value in the range [0, 1])", this), + diskUtilization("disk_utilization", "", "The relative amount of disk used compared to the disk resource limit", this), memory("memory", "", "The relative amount of memory used by this process (value in the range [0, 1])", this), + memoryUtilization("memory_utilization", "", "The relative amount of memory used compared to the memory resource limit", this), memoryMappings("memory_mappings", "", "The number of mapped memory areas", this), openFileDescriptors("open_file_descriptors", "", "The number of open files", this), feedingBlocked("feeding_blocked", "", "Whether feeding is blocked due to resource limits being reached (value is either 0 or 1)", this) diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h index 698ba8029bd..497a0fe2ba0 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h @@ -12,7 +12,9 @@ namespace proton { struct ResourceUsageMetrics : metrics::MetricSet { metrics::DoubleValueMetric disk; + metrics::DoubleValueMetric diskUtilization; metrics::DoubleValueMetric memory; + metrics::DoubleValueMetric memoryUtilization; metrics::LongValueMetric memoryMappings; metrics::LongValueMetric openFileDescriptors; metrics::LongValueMetric feedingBlocked; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index 32b2b6aaba6..d53adf19f76 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -209,18 +209,11 @@ DiskMemUsageFilter::getConfig() const return _config; } -double -DiskMemUsageFilter::getMemoryUsedRatio() const -{ - Guard guard(_lock); - return getMemoryUsedRatio(guard); -} - -double -DiskMemUsageFilter::getDiskUsedRatio() const +DiskMemUsageState +DiskMemUsageFilter::usageState() const { Guard guard(_lock); - return getDiskUsedRatio(guard); + return _dmstate; } bool diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index 4906348a54d..982e3bba6fd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -64,8 +64,7 @@ public: uint64_t getDiskUsedSize() const; Config getConfig() const; const HwInfo &getHwInfo() const { return _hwInfo; } - double getMemoryUsedRatio() const; - double getDiskUsedRatio() const; + DiskMemUsageState usageState() const; bool acceptWriteOperation() const override; State getAcceptState() const override; void addDiskMemUsageListener(IDiskMemUsageListener *listener) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 6de5410470f..ecdcf1c9c0f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -704,8 +704,11 @@ Proton::updateMetrics(const vespalib::MonitorGuard &) ContentProtonMetrics &metrics = _metricsEngine->root(); metrics.transactionLog.update(_tls->getTransLogServer()->getDomainStats()); const DiskMemUsageFilter &usageFilter = _diskMemUsageSampler->writeFilter(); - metrics.resourceUsage.disk.set(usageFilter.getDiskUsedRatio()); - metrics.resourceUsage.memory.set(usageFilter.getMemoryUsedRatio()); + DiskMemUsageState usageState = usageFilter.usageState(); + metrics.resourceUsage.disk.set(usageState.diskState().usage()); + metrics.resourceUsage.diskUtilization.set(usageState.diskState().utilization()); + metrics.resourceUsage.memory.set(usageState.memoryState().usage()); + metrics.resourceUsage.memoryUtilization.set(usageState.memoryState().utilization()); metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount()); metrics.resourceUsage.openFileDescriptors.set(countOpenFiles()); metrics.resourceUsage.feedingBlocked.set((usageFilter.acceptWriteOperation() ? 0.0 : 1.0)); diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp index 33b37649ed2..6f1fd8b90e0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp @@ -33,21 +33,23 @@ void ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool full) const { Cursor &object = inserter.insertObject(); + DiskMemUsageState usageState = _usageFilter.usageState(); if (full) { - DiskMemUsageFilter::Config config = _usageFilter.getConfig(); Cursor &disk = object.setObject("disk"); - disk.setDouble("usedRatio", _usageFilter.getDiskUsedRatio()); - disk.setDouble("usedLimit", config._diskLimit); + disk.setDouble("usage", usageState.diskState().usage()); + disk.setDouble("limit", usageState.diskState().limit()); + disk.setDouble("utilization", usageState.diskState().utilization()); convertDiskStatsToSlime(_usageFilter.getHwInfo(), _usageFilter.getDiskUsedSize(), disk.setObject("stats")); Cursor &memory = object.setObject("memory"); - memory.setDouble("usedRatio", _usageFilter.getMemoryUsedRatio()); - memory.setDouble("usedLimit", config._memoryLimit); + memory.setDouble("usage", usageState.memoryState().usage()); + memory.setDouble("limit", usageState.memoryState().limit()); + memory.setDouble("utilization", usageState.memoryState().utilization()); memory.setLong("physicalMemory", _usageFilter.getHwInfo().memory().sizeBytes()); convertMemoryStatsToSlime(_usageFilter.getMemoryStats(), memory.setObject("stats")); } else { - object.setDouble("disk", _usageFilter.getDiskUsedRatio()); - object.setDouble("memory", _usageFilter.getMemoryUsedRatio()); + object.setDouble("disk", usageState.diskState().usage()); + object.setDouble("memory", usageState.memoryState().usage()); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h index 7ac127c6b25..29f9959aea9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h @@ -5,7 +5,10 @@ namespace proton { /** - * Class representing the state of an resource (e.g. disk or memory) with its limit and current usage. + * Class representing the state of an resource (e.g. disk or memory) with its limit and current usage: + * - usage: How much of this resource is currently used (number between 0 and 1). + * - limit: How much of this resource is allowed to use (number between 0 and 1). + * - utilization: How much of the allowed part of this resource is used (usage/limit). */ class ResourceUsageState { @@ -33,6 +36,7 @@ public: } double limit() const { return _limit; } double usage() const { return _usage; } + double utilization() const { return _usage/_limit; } bool aboveLimit() const { return aboveLimit(1.0); } |