diff options
author | Geir Storli <geirst@oath.com> | 2017-09-20 08:36:41 +0000 |
---|---|---|
committer | Geir Storli <geirst@oath.com> | 2017-09-20 08:36:41 +0000 |
commit | eda7f805e9c868c8500d46333431555d06410d13 (patch) | |
tree | 5ee0db0937bc3b6ffff3b0744d0078299e786de6 /searchcore | |
parent | def104b531e19af93d388aa760326a9da2bcbf60 (diff) |
Use HwInfo for disk and memory size in DiskMemUsageSampler and sample used disk size under vespa home dir.
This should fix sampling of disk usage when using docker.
Diffstat (limited to 'searchcore')
7 files changed, 92 insertions, 79 deletions
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp index 5d8a647f675..05d53c0bf18 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp @@ -1,9 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/server/disk_mem_usage_filter.h> using proton::DiskMemUsageFilter; +using proton::HwInfo; namespace fs = std::experimental::filesystem; @@ -16,9 +18,9 @@ struct Fixture using Config = DiskMemUsageFilter::Config; Fixture() - : _filter(64 * 1024 * 1024) + : _filter(HwInfo(100, false, 64 * 1024 * 1024)) { - _filter.setDiskStats({.capacity = 100, .free = 100, .available=100}); + _filter.setDiskStats(0); _filter.setMemoryStats(vespalib::ProcessMemoryStats(10000000, 10000001, 10000002, @@ -41,7 +43,7 @@ struct Fixture } void triggerDiskLimit() { - _filter.setDiskStats({.capacity = 100, .free = 20, .available=10}); + _filter.setDiskStats(90); } void triggerMemoryLimit() @@ -76,7 +78,7 @@ TEST_F("Check that disk limit can be reached", Fixture) "action: \"add more content nodes\", " "reason: \"disk used (0.9) > disk limit (0.8)\", " "stats: { " - "capacity: 100, free: 20, available: 10, diskUsed: 0.9, diskLimit: 0.8}}"); + "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); } TEST_F("Check that memory limit can be reached", Fixture) @@ -108,7 +110,7 @@ TEST_F("Check that both disk limit and memory limit can be reached", Fixture) "action: \"add more content nodes\", " "reason: \"disk used (0.9) > disk limit (0.8)\", " "stats: { " - "capacity: 100, free: 20, available: 10, diskUsed: 0.9, diskLimit: 0.8}}"); + "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index d489f477df2..307eed94095 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -3,6 +3,7 @@ #include "disk_mem_usage_filter.h" #include "i_disk_mem_usage_listener.h" #include <vespa/log/log.h> +#include <vespa/searchcore/proton/common/hw_info.h> LOG_SETUP(".proton.server.disk_mem_usage_filter"); @@ -43,12 +44,12 @@ void makeDiskStatsMessage(std::ostream &os, double diskUsed, double diskLimit, - const DiskMemUsageFilter::space_info &diskStats) + const HwInfo &hwInfo, + uint64_t usedDiskSizeBytes) { os << "stats: { "; - os << "capacity: " << diskStats.capacity << ", "; - os << "free: " << diskStats.free << ", "; - os << "available: " << diskStats.available << ", "; + os << "capacity: " << hwInfo.diskSizeBytes() << ", "; + os << "used: " << usedDiskSizeBytes << ", "; os << "diskUsed: " << diskUsed << ", "; os << "diskLimit: " << diskLimit << "}"; } @@ -57,31 +58,31 @@ void makeDiskLimitMessage(std::ostream &os, double diskUsed, double diskLimit, - const DiskMemUsageFilter::space_info &diskStats) + const HwInfo &hwInfo, + uint64_t usedDiskSizeBytes) { os << "diskLimitReached: { "; os << "action: \"add more content nodes\", "; os << "reason: \"disk used (" << diskUsed << ") > disk limit (" << diskLimit << ")\", "; - makeDiskStatsMessage(os, diskUsed, diskLimit, diskStats); + makeDiskStatsMessage(os, diskUsed, diskLimit, hwInfo, usedDiskSizeBytes); os << "}"; } - vespalib::string makeUnblockingMessage(double memoryUsed, double memoryLimit, const vespalib::ProcessMemoryStats &memoryStats, - uint64_t physicalMemory, + const HwInfo &hwInfo, double diskUsed, double diskLimit, - const DiskMemUsageFilter::space_info &diskStats) + uint64_t usedDiskSizeBytes) { std::ostringstream os; os << "memoryLimitOK: { "; - makeMemoryStatsMessage(os, memoryUsed, memoryLimit, memoryStats, physicalMemory); + makeMemoryStatsMessage(os, memoryUsed, memoryLimit, memoryStats, hwInfo.memorySizeBytes()); os << "}, "; os << "diskLimitOK: { "; - makeDiskStatsMessage(os, diskUsed, diskLimit, diskStats); + makeDiskStatsMessage(os, diskUsed, diskLimit, hwInfo, usedDiskSizeBytes); os << "}"; return os.str(); } @@ -97,7 +98,7 @@ DiskMemUsageFilter::recalcState(const Guard &guard) if (memoryUsed > _config._memoryLimit) { hasMessage = true; makeMemoryLimitMessage(message, memoryUsed, - _config._memoryLimit, _memoryStats, _physicalMemory); + _config._memoryLimit, _memoryStats, _hwInfo.memorySizeBytes()); } double diskUsed = getDiskUsedRatio(guard); if (diskUsed > _config._diskLimit) { @@ -105,7 +106,7 @@ DiskMemUsageFilter::recalcState(const Guard &guard) message << ", "; } hasMessage = true; - makeDiskLimitMessage(message, diskUsed, _config._diskLimit, _diskStats); + makeDiskLimitMessage(message, diskUsed, _config._diskLimit, _hwInfo, _usedDiskSizeBytes); } if (hasMessage) { if (_acceptWrite) { @@ -118,10 +119,10 @@ DiskMemUsageFilter::recalcState(const Guard &guard) vespalib::string unblockMsg = makeUnblockingMessage(memoryUsed, _config._memoryLimit, _memoryStats, - _physicalMemory, + _hwInfo, diskUsed, _config._diskLimit, - _diskStats); + _usedDiskSizeBytes); LOG(info, "Write operations are now un-blocked: '%s'", unblockMsg.c_str()); } _state = State(); @@ -137,23 +138,23 @@ DiskMemUsageFilter::getMemoryUsedRatio(const Guard &guard) const { (void) guard; uint64_t unscaledMemoryUsed = _memoryStats.getAnonymousRss(); - return static_cast<double>(unscaledMemoryUsed) / _physicalMemory; + return static_cast<double>(unscaledMemoryUsed) / _hwInfo.memorySizeBytes(); } double DiskMemUsageFilter::getDiskUsedRatio(const Guard &guard) const { (void) guard; - double availableDiskSpaceRatio = static_cast<double>(_diskStats.available) / - static_cast<double>(_diskStats.capacity); - return 1.0 - availableDiskSpaceRatio; + double usedDiskSpaceRatio = static_cast<double>(_usedDiskSizeBytes) / + static_cast<double>(_hwInfo.diskSizeBytes()); + return usedDiskSpaceRatio; } -DiskMemUsageFilter::DiskMemUsageFilter(uint64_t physicalMemory_in) +DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) : _lock(), + _hwInfo(hwInfo), _memoryStats(), - _physicalMemory(physicalMemory_in), - _diskStats(), + _usedDiskSizeBytes(), _config(), _state(), _acceptWrite(true), @@ -172,10 +173,10 @@ DiskMemUsageFilter::setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in) } void -DiskMemUsageFilter::setDiskStats(space_info diskStats_in) +DiskMemUsageFilter::setDiskStats(uint64_t usedDiskSizeBytes) { Guard guard(_lock); - _diskStats = diskStats_in; + _usedDiskSizeBytes = usedDiskSizeBytes; recalcState(guard); } @@ -194,11 +195,11 @@ DiskMemUsageFilter::getMemoryStats() const return _memoryStats; } -DiskMemUsageFilter::space_info +uint64_t DiskMemUsageFilter::getDiskStats() const { Guard guard(_lock); - return _diskStats; + return _usedDiskSizeBytes; } DiskMemUsageFilter::Config diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index c06d2e2ab0a..1083b969edd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -4,12 +4,12 @@ #include "i_disk_mem_usage_notifier.h" #include "disk_mem_usage_state.h" +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/vespalib/util/process_memory_stats.h> -#include <mutex> #include <atomic> #include <experimental/filesystem> - +#include <mutex> namespace proton { @@ -39,10 +39,10 @@ public: }; private: - mutable Mutex _lock; // protect _memoryStats, _diskStats, _config, _state + mutable Mutex _lock; // protect _memoryStats, _usedDiskSizeBytes, _config, _state + HwInfo _hwInfo; vespalib::ProcessMemoryStats _memoryStats; - uint64_t _physicalMemory; - space_info _diskStats; + uint64_t _usedDiskSizeBytes; Config _config; State _state; std::atomic<bool> _acceptWrite; @@ -55,15 +55,15 @@ private: void notifyDiskMemUsage(const Guard &guard, DiskMemUsageState state); public: - DiskMemUsageFilter(uint64_t physicalMememory_in); + DiskMemUsageFilter(const HwInfo &hwInfo); ~DiskMemUsageFilter(); void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in); - void setDiskStats(space_info diskStats_in); + void setDiskStats(uint64_t usedDiskSizeBytes); void setConfig(Config config); vespalib::ProcessMemoryStats getMemoryStats() const; - space_info getDiskStats() const; + uint64_t getDiskStats() const; Config getConfig() const; - uint64_t getPhysicalMemory() const { return _physicalMemory; } + const HwInfo &getHwInfo() const { return _hwInfo; } double getMemoryUsedRatio() const; double getDiskUsedRatio() const; bool acceptWriteOperation() const override; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp index fb7d712db9d..e45f83fc900 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp @@ -3,30 +3,16 @@ #include "disk_mem_usage_sampler.h" #include <vespa/vespalib/util/timer.h> #include <vespa/searchlib/common/lambdatask.h> +#include <experimental/filesystem> #include <unistd.h> using search::makeLambdaTask; namespace proton { -namespace { - -uint64_t getPhysicalMemoryBytes() -{ - // TODO: Temporal workaround for Docker nodes. Remove when this is part of proton.cfg instead. - if (const char *memoryEnv = std::getenv("VESPA_TOTAL_MEMORY_MB")) { - uint64_t physicalMemoryMB = atoll(memoryEnv); - return physicalMemoryMB * 1024u * 1024u; - } else { - return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); - } -} - -} // namespace proton:<anonymous> - DiskMemUsageSampler::DiskMemUsageSampler(const std::string &path_in, const Config &config) - : _filter(getPhysicalMemoryBytes()), + : _filter(config.hwInfo), _path(path_in), _sampleInterval(60.0), _periodicTimer() @@ -43,8 +29,8 @@ void DiskMemUsageSampler::setConfig(const Config &config) { _periodicTimer.reset(); - _filter.setConfig(config._filterConfig); - _sampleInterval = config._sampleInterval; + _filter.setConfig(config.filterConfig); + _sampleInterval = config.sampleInterval; sampleUsage(); _periodicTimer = std::make_unique<vespalib::Timer>(); _periodicTimer->scheduleAtFixedRate(makeLambdaTask([this]() @@ -59,10 +45,26 @@ DiskMemUsageSampler::sampleUsage() sampleDiskUsage(); } +namespace { + +uint64_t +sampleDiskUsageInDirectory(const std::experimental::filesystem::path &path) +{ + uint64_t result = 0; + for (const auto &elem : std::experimental::filesystem::recursive_directory_iterator(path)) { + if (!std::experimental::filesystem::is_directory(elem.path())) { + result += std::experimental::filesystem::file_size(elem.path()); + } + } + return result; +} + +} + void DiskMemUsageSampler::sampleDiskUsage() { - _filter.setDiskStats(std::experimental::filesystem::space(_path)); + _filter.setDiskStats(sampleDiskUsageInDirectory(_path)); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h index 198f111f052..4ed48613f6a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h @@ -22,19 +22,24 @@ class DiskMemUsageSampler { void sampleMemoryUsage(); public: struct Config { - DiskMemUsageFilter::Config _filterConfig; - double _sampleInterval; - public: + DiskMemUsageFilter::Config filterConfig; + double sampleInterval; + HwInfo hwInfo; + Config() - : _filterConfig(), - _sampleInterval(60.0) + : filterConfig(), + sampleInterval(60.0), + hwInfo() { } - Config(double memoryLimit_in, double diskLimit_in, - double sampleInterval_in) - : _filterConfig(memoryLimit_in, diskLimit_in), - _sampleInterval(sampleInterval_in) + Config(double memoryLimit_in, + double diskLimit_in, + double sampleInterval_in, + const HwInfo &hwInfo_in) + : filterConfig(memoryLimit_in, diskLimit_in), + sampleInterval(sampleInterval_in), + hwInfo(hwInfo_in) { } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index ba54cdf847c..80c2ce8cedf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -32,6 +32,8 @@ #include <vespa/searchlib/expression/forcelink.hpp> #include <vespa/log/log.h> +#include <vespa/defaults.h> + LOG_SETUP(".proton.server.proton"); using document::DocumentTypeRepo; @@ -75,12 +77,13 @@ setFS4Compression(const ProtonConfig & proton) } DiskMemUsageSampler::Config -diskMemUsageSamplerConfig(const ProtonConfig &proton) +diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo) { return DiskMemUsageSampler::Config( proton.writefilter.memorylimit, proton.writefilter.disklimit, - proton.writefilter.sampleinterval); + proton.writefilter.sampleinterval, + hwInfo); } } @@ -234,9 +237,10 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) samplerCfg); _hwInfo = _hwInfoSampler->hwInfo(); setFS4Compression(protonConfig); + vespalib::string vespaHomeDir = vespa::Defaults::vespaHome(); _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler> - (protonConfig.basedir, - diskMemUsageSamplerConfig(protonConfig)); + (vespaHomeDir, + diskMemUsageSamplerConfig(protonConfig, _hwInfo)); _metricsEngine.reset(new MetricsEngine()); _metricsEngine->addMetricsHook(_metricsHook); @@ -344,7 +348,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot) protonConfig.search.memory.limiter.minhits); const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP(); - _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig)); + _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, _hwInfo)); if (_memoryFlushConfigUpdater) { _memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory); _flushEngine->kick(); diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp index afe9f6b85d3..7bd01760141 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp @@ -9,11 +9,10 @@ using namespace vespalib::slime; namespace proton { void -convertDiskStatsToSlime(const DiskMemUsageFilter::space_info &stats, Cursor &object) +convertDiskStatsToSlime(const HwInfo &hwInfo, uint64_t usedDiskSizeBytes, Cursor &object) { - object.setLong("capacity", stats.capacity); - object.setLong("free", stats.free); - object.setLong("available", stats.available); + object.setLong("capacity", hwInfo.diskSizeBytes()); + object.setLong("used", usedDiskSizeBytes); } void @@ -39,12 +38,12 @@ ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool Cursor &disk = object.setObject("disk"); disk.setDouble("usedRatio", _usageFilter.getDiskUsedRatio()); disk.setDouble("usedLimit", config._diskLimit); - convertDiskStatsToSlime(_usageFilter.getDiskStats(), disk.setObject("stats")); + convertDiskStatsToSlime(_usageFilter.getHwInfo(), _usageFilter.getDiskStats(), disk.setObject("stats")); Cursor &memory = object.setObject("memory"); memory.setDouble("usedRatio", _usageFilter.getMemoryUsedRatio()); memory.setDouble("usedLimit", config._memoryLimit); - memory.setLong("physicalMemory", _usageFilter.getPhysicalMemory()); + memory.setLong("physicalMemory", _usageFilter.getHwInfo().memorySizeBytes()); convertMemoryStatsToSlime(_usageFilter.getMemoryStats(), memory.setObject("stats")); } else { object.setDouble("disk", _usageFilter.getDiskUsedRatio()); |