diff options
author | Tor Brede Vekterli <vekterli@yahoo-inc.com> | 2017-12-04 11:31:29 +0000 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@yahoo-inc.com> | 2017-12-04 13:06:46 +0000 |
commit | 952a877611d657cfa0166b14699c8731b18f7587 (patch) | |
tree | c1b4330b9dc512c3ffbf845e626857ddcd801982 /memfilepersistence | |
parent | d164fbb93e277ef23ab610320a7cf8556e3c036e (diff) |
Remove memory manager component from content layer
We already have resource utilization tracking in both MessageBus
and the search core. The memory manager has never been auto-scaled
based on the hardware present and adds a _lot_ of complexity without
having any known instances where it has actually saved the day.
Removing it also removes a mutex on the message hot path.
If we need such functionality in the future, should design a lock-free
solution.
Cleanup
Diffstat (limited to 'memfilepersistence')
7 files changed, 4 insertions, 45 deletions
diff --git a/memfilepersistence/src/tests/spi/memcachetest.cpp b/memfilepersistence/src/tests/spi/memcachetest.cpp index 73815df563a..5e9f1a28225 100644 --- a/memfilepersistence/src/tests/spi/memcachetest.cpp +++ b/memfilepersistence/src/tests/spi/memcachetest.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/memfilepersistence/memfile/memfilecache.h> -#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h> #include <tests/spi/memfiletestutils.h> @@ -41,7 +40,6 @@ private: framework::defaultimplementation::ComponentRegisterImpl::UP _register; framework::Component::UP _component; FakeClock::UP _clock; - framework::defaultimplementation::MemoryManager::UP _memoryManager; std::unique_ptr<MemFilePersistenceMetrics> _metrics; std::unique_ptr<MemFileCache> _cache; @@ -113,18 +111,10 @@ private: new framework::defaultimplementation::ComponentRegisterImpl); _clock.reset(new FakeClock); _register->setClock(*_clock); - _memoryManager.reset( - new framework::defaultimplementation::MemoryManager( - framework::defaultimplementation::AllocationLogic::UP( - new framework::defaultimplementation::SimpleMemoryLogic( - *_clock, maxMemory * 2)))); - _register->setMemoryManager(*_memoryManager); _component.reset(new framework::Component(*_register, "testcomponent")); _metrics.reset(new MemFilePersistenceMetrics(*_component)); _cache.reset(new MemFileCache(*_register, _metrics->_cache)); setCacheSize(maxMemory); - _memoryManager->registerAllocationType(framework::MemoryAllocationType( - "steal", framework::MemoryAllocationType::FORCE_ALLOCATE)); } public: @@ -133,7 +123,6 @@ public: _metrics.reset(0); _component.reset(0); _register.reset(0); - _memoryManager.reset(0); _clock.reset(0); } }; diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.cpp b/memfilepersistence/src/tests/spi/memfiletestutils.cpp index bb35aa4d03e..cd14e989220 100644 --- a/memfilepersistence/src/tests/spi/memfiletestutils.cpp +++ b/memfilepersistence/src/tests/spi/memfiletestutils.cpp @@ -5,7 +5,6 @@ #include <tests/spi/memfiletestutils.h> #include <tests/spi/simulatedfailurefile.h> #include <vespa/memfilepersistence/memfile/memfilecache.h> -#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/persistence/spi/test.h> @@ -72,12 +71,6 @@ MemFileTestUtils::setupDisks(uint32_t numDisks) { new framework::defaultimplementation::ComponentRegisterImpl); _clock.reset(new FakeClock); _componentRegister->setClock(*_clock); - _memoryManager.reset( - new framework::defaultimplementation::MemoryManager( - framework::defaultimplementation::AllocationLogic::UP( - new framework::defaultimplementation::SimpleMemoryLogic( - *_clock, 1024 * 1024 * 1024)))); - _componentRegister->setMemoryManager(*_memoryManager); _env.reset(new MemFileTestEnvironment(numDisks, *_componentRegister, *getTypeRepo())); diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.h b/memfilepersistence/src/tests/spi/memfiletestutils.h index dde73c88164..450d87451b9 100644 --- a/memfilepersistence/src/tests/spi/memfiletestutils.h +++ b/memfilepersistence/src/tests/spi/memfiletestutils.h @@ -20,8 +20,6 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> -#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> - namespace storage { namespace memfile { @@ -70,7 +68,6 @@ private: document::BucketIdFactory _bucketIdFactory; framework::defaultimplementation::ComponentRegisterImpl::UP _componentRegister; FakeClock::UP _clock; - framework::defaultimplementation::MemoryManager::UP _memoryManager; std::unique_ptr<MemFileTestEnvironment> _env; public: @@ -82,7 +79,6 @@ public: void tearDown() override{ _env.reset(); _componentRegister.reset(); - _memoryManager.reset(); _clock.reset(); } diff --git a/memfilepersistence/src/tests/spi/providerconformancetest.cpp b/memfilepersistence/src/tests/spi/providerconformancetest.cpp index db02e619782..625549f0008 100644 --- a/memfilepersistence/src/tests/spi/providerconformancetest.cpp +++ b/memfilepersistence/src/tests/spi/providerconformancetest.cpp @@ -2,7 +2,6 @@ #include "memfiletestutils.h" #include <vespa/persistence/conformancetest/conformancetest.h> -#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h> namespace storage { namespace memfile { @@ -11,19 +10,13 @@ struct ProviderConformanceTest : public spi::ConformanceTest { struct Factory : public PersistenceFactory { framework::defaultimplementation::ComponentRegisterImpl _compRegister; framework::defaultimplementation::RealClock _clock; - framework::defaultimplementation::MemoryManager _memoryManager; std::unique_ptr<MemFileCache> cache; Factory() : _compRegister(), - _clock(), - _memoryManager( - framework::defaultimplementation::AllocationLogic::UP( - new framework::defaultimplementation::SimpleMemoryLogic( - _clock, 1024 * 1024 * 1024))) + _clock() { _compRegister.setClock(_clock); - _compRegister.setMemoryManager(_memoryManager); } spi::PersistenceProvider::UP diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp index 5f24d28a309..cfd73d1a7cb 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp @@ -31,7 +31,7 @@ MemFileCache::returnToCache(MemFileCache::Entry& entry) MemoryUsage newUsage = entry._file.getCacheSize(); - if (_memoryToken->getSize() == 0 || newUsage.sum() == 0) { + if (_cacheLimit.sum() == 0 || newUsage.sum() == 0) { entry._file.flushToDisk(); eraseNoLock(id); return; @@ -155,10 +155,6 @@ MemFileCache::MemFileCache(framework::ComponentRegister& componentRegister, MemFilePersistenceCacheMetrics& metrics) : Component(componentRegister, "memfilecache"), _lastUsedCounter(0), - _allocationType(getMemoryManager().registerAllocationType( - framework::MemoryAllocationType( - "memfilecache", framework::MemoryAllocationType::CACHE))), - _memoryToken(getMemoryManager().allocate(_allocationType, 0, 0, 200)), _metrics(metrics), _bodyEvicter(_metrics.body_evictions), _headerEvicter(_metrics.header_evictions), @@ -172,7 +168,6 @@ MemFileCache::setCacheSize(MemoryUsage cacheSize) vespalib::LockGuard lock(_cacheLock); _cacheLimit = cacheSize; - _memoryToken->resize(std::min(_memoryToken->getSize(), _cacheLimit.sum()), _cacheLimit.sum()); evictWhileFull(); } @@ -491,7 +486,7 @@ MemFileCache::Statistics MemFileCache::getCacheStats() const { vespalib::LockGuard lock(_cacheLock); - return Statistics(_memoryUsage, _memoryToken->getSize(), _entries.size()); + return Statistics(_memoryUsage, _cacheLimit.sum(), _entries.size()); } void diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h index e45f8f7f1f6..36e6e6b641a 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h @@ -21,7 +21,6 @@ #include <boost/multi_index/mem_fun.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/generic/component/component.h> @@ -191,8 +190,6 @@ private: LRUCache _entries; uint64_t _lastUsedCounter; - const framework::MemoryAllocationType& _allocationType; - framework::MemoryToken::UP _memoryToken; MemFilePersistenceCacheMetrics& _metrics; @@ -273,7 +270,6 @@ public: /** * Used for unit testing only. */ - framework::MemoryToken& getMemoryToken() { return *_memoryToken; } const MemFilePersistenceCacheMetrics& getMetrics() const { return _metrics; } @@ -283,7 +279,7 @@ public: */ void setCacheSize(MemoryUsage limits); - uint64_t getCacheSize() { return _memoryToken->getSize(); } + uint64_t getCacheSize() { return _cacheLimit.sum(); } /** * NOTE: takes lock, never call from within memfilecache code. diff --git a/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp b/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp index 1dccd255ebb..9781c28823d 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp @@ -11,7 +11,6 @@ #include <vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> -#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h> #include <vespa/vespalib/util/programoptions.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/config/subscription/configuri.h> @@ -139,7 +138,6 @@ namespace { framework::defaultimplementation::ComponentRegisterImpl _compReg; framework::Component _component; framework::defaultimplementation::RealClock _clock; - framework::defaultimplementation::NoMemoryManager _memoryMan; MemFilePersistenceMetrics _metrics; MemFilePersistenceThreadMetrics* _threadMetrics; std::unique_ptr<MemFileCache> _cache; @@ -177,7 +175,6 @@ namespace { _docType("foo", 1) { _compReg.setClock(_clock); - _compReg.setMemoryManager(_memoryMan); _cache.reset(new MemFileCache(_compReg, _metrics._cache)); if (documentConfigId == 0) { _repo.reset(new DocumentTypeRepo(_docType)); |