diff options
78 files changed, 26 insertions, 3474 deletions
diff --git a/memfilepersistence/src/tests/spi/memcachetest.cpp b/memfilepersistence/src/tests/spi/memcachetest.cpp index 73815df563a..5e9f1a28225 100644 --- a/memfilepersistence/src/tests/spi/memcachetest.cpp +++ b/memfilepersistence/src/tests/spi/memcachetest.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/memfilepersistence/memfile/memfilecache.h> -#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h> #include <tests/spi/memfiletestutils.h> @@ -41,7 +40,6 @@ private: framework::defaultimplementation::ComponentRegisterImpl::UP _register; framework::Component::UP _component; FakeClock::UP _clock; - framework::defaultimplementation::MemoryManager::UP _memoryManager; std::unique_ptr<MemFilePersistenceMetrics> _metrics; std::unique_ptr<MemFileCache> _cache; @@ -113,18 +111,10 @@ private: new framework::defaultimplementation::ComponentRegisterImpl); _clock.reset(new FakeClock); _register->setClock(*_clock); - _memoryManager.reset( - new framework::defaultimplementation::MemoryManager( - framework::defaultimplementation::AllocationLogic::UP( - new framework::defaultimplementation::SimpleMemoryLogic( - *_clock, maxMemory * 2)))); - _register->setMemoryManager(*_memoryManager); _component.reset(new framework::Component(*_register, "testcomponent")); _metrics.reset(new MemFilePersistenceMetrics(*_component)); _cache.reset(new MemFileCache(*_register, _metrics->_cache)); setCacheSize(maxMemory); - _memoryManager->registerAllocationType(framework::MemoryAllocationType( - "steal", framework::MemoryAllocationType::FORCE_ALLOCATE)); } public: @@ -133,7 +123,6 @@ public: _metrics.reset(0); _component.reset(0); _register.reset(0); - _memoryManager.reset(0); _clock.reset(0); } }; diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.cpp b/memfilepersistence/src/tests/spi/memfiletestutils.cpp index bb35aa4d03e..cd14e989220 100644 --- a/memfilepersistence/src/tests/spi/memfiletestutils.cpp +++ b/memfilepersistence/src/tests/spi/memfiletestutils.cpp @@ -5,7 +5,6 @@ #include <tests/spi/memfiletestutils.h> #include <tests/spi/simulatedfailurefile.h> #include <vespa/memfilepersistence/memfile/memfilecache.h> -#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/persistence/spi/test.h> @@ -72,12 +71,6 @@ MemFileTestUtils::setupDisks(uint32_t numDisks) { new framework::defaultimplementation::ComponentRegisterImpl); _clock.reset(new FakeClock); _componentRegister->setClock(*_clock); - _memoryManager.reset( - new framework::defaultimplementation::MemoryManager( - framework::defaultimplementation::AllocationLogic::UP( - new framework::defaultimplementation::SimpleMemoryLogic( - *_clock, 1024 * 1024 * 1024)))); - _componentRegister->setMemoryManager(*_memoryManager); _env.reset(new MemFileTestEnvironment(numDisks, *_componentRegister, *getTypeRepo())); diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.h b/memfilepersistence/src/tests/spi/memfiletestutils.h index dde73c88164..450d87451b9 100644 --- a/memfilepersistence/src/tests/spi/memfiletestutils.h +++ b/memfilepersistence/src/tests/spi/memfiletestutils.h @@ -20,8 +20,6 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> -#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> - namespace storage { namespace memfile { @@ -70,7 +68,6 @@ private: document::BucketIdFactory _bucketIdFactory; framework::defaultimplementation::ComponentRegisterImpl::UP _componentRegister; FakeClock::UP _clock; - framework::defaultimplementation::MemoryManager::UP _memoryManager; std::unique_ptr<MemFileTestEnvironment> _env; public: @@ -82,7 +79,6 @@ public: void tearDown() override{ _env.reset(); _componentRegister.reset(); - _memoryManager.reset(); _clock.reset(); } diff --git a/memfilepersistence/src/tests/spi/providerconformancetest.cpp b/memfilepersistence/src/tests/spi/providerconformancetest.cpp index db02e619782..625549f0008 100644 --- a/memfilepersistence/src/tests/spi/providerconformancetest.cpp +++ b/memfilepersistence/src/tests/spi/providerconformancetest.cpp @@ -2,7 +2,6 @@ #include "memfiletestutils.h" #include <vespa/persistence/conformancetest/conformancetest.h> -#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h> namespace storage { namespace memfile { @@ -11,19 +10,13 @@ struct ProviderConformanceTest : public spi::ConformanceTest { struct Factory : public PersistenceFactory { framework::defaultimplementation::ComponentRegisterImpl _compRegister; framework::defaultimplementation::RealClock _clock; - framework::defaultimplementation::MemoryManager _memoryManager; std::unique_ptr<MemFileCache> cache; Factory() : _compRegister(), - _clock(), - _memoryManager( - framework::defaultimplementation::AllocationLogic::UP( - new framework::defaultimplementation::SimpleMemoryLogic( - _clock, 1024 * 1024 * 1024))) + _clock() { _compRegister.setClock(_clock); - _compRegister.setMemoryManager(_memoryManager); } spi::PersistenceProvider::UP diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp index 5f24d28a309..cfd73d1a7cb 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp @@ -31,7 +31,7 @@ MemFileCache::returnToCache(MemFileCache::Entry& entry) MemoryUsage newUsage = entry._file.getCacheSize(); - if (_memoryToken->getSize() == 0 || newUsage.sum() == 0) { + if (_cacheLimit.sum() == 0 || newUsage.sum() == 0) { entry._file.flushToDisk(); eraseNoLock(id); return; @@ -155,10 +155,6 @@ MemFileCache::MemFileCache(framework::ComponentRegister& componentRegister, MemFilePersistenceCacheMetrics& metrics) : Component(componentRegister, "memfilecache"), _lastUsedCounter(0), - _allocationType(getMemoryManager().registerAllocationType( - framework::MemoryAllocationType( - "memfilecache", framework::MemoryAllocationType::CACHE))), - _memoryToken(getMemoryManager().allocate(_allocationType, 0, 0, 200)), _metrics(metrics), _bodyEvicter(_metrics.body_evictions), _headerEvicter(_metrics.header_evictions), @@ -172,7 +168,6 @@ MemFileCache::setCacheSize(MemoryUsage cacheSize) vespalib::LockGuard lock(_cacheLock); _cacheLimit = cacheSize; - _memoryToken->resize(std::min(_memoryToken->getSize(), _cacheLimit.sum()), _cacheLimit.sum()); evictWhileFull(); } @@ -491,7 +486,7 @@ MemFileCache::Statistics MemFileCache::getCacheStats() const { vespalib::LockGuard lock(_cacheLock); - return Statistics(_memoryUsage, _memoryToken->getSize(), _entries.size()); + return Statistics(_memoryUsage, _cacheLimit.sum(), _entries.size()); } void diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h index e45f8f7f1f6..36e6e6b641a 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h @@ -21,7 +21,6 @@ #include <boost/multi_index/mem_fun.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/generic/component/component.h> @@ -191,8 +190,6 @@ private: LRUCache _entries; uint64_t _lastUsedCounter; - const framework::MemoryAllocationType& _allocationType; - framework::MemoryToken::UP _memoryToken; MemFilePersistenceCacheMetrics& _metrics; @@ -273,7 +270,6 @@ public: /** * Used for unit testing only. */ - framework::MemoryToken& getMemoryToken() { return *_memoryToken; } const MemFilePersistenceCacheMetrics& getMetrics() const { return _metrics; } @@ -283,7 +279,7 @@ public: */ void setCacheSize(MemoryUsage limits); - uint64_t getCacheSize() { return _memoryToken->getSize(); } + uint64_t getCacheSize() { return _cacheLimit.sum(); } /** * NOTE: takes lock, never call from within memfilecache code. diff --git a/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp b/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp index 1dccd255ebb..9781c28823d 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp @@ -11,7 +11,6 @@ #include <vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> -#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h> #include <vespa/vespalib/util/programoptions.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/config/subscription/configuri.h> @@ -139,7 +138,6 @@ namespace { framework::defaultimplementation::ComponentRegisterImpl _compReg; framework::Component _component; framework::defaultimplementation::RealClock _clock; - framework::defaultimplementation::NoMemoryManager _memoryMan; MemFilePersistenceMetrics _metrics; MemFilePersistenceThreadMetrics* _threadMetrics; std::unique_ptr<MemFileCache> _cache; @@ -177,7 +175,6 @@ namespace { _docType("foo", 1) { _compReg.setClock(_clock); - _compReg.setMemoryManager(_memoryMan); _cache.reset(new MemFileCache(_compReg, _metrics._cache)); if (documentConfigId == 0) { _repo.reset(new DocumentTypeRepo(_docType)); diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 24d5a2d27f2..5c24d4adb3b 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -35,7 +35,6 @@ vespa_define_module( src/vespa/storage/distributor/operations/external src/vespa/storage/distributor/operations/idealstate src/vespa/storage/frameworkimpl/component - src/vespa/storage/frameworkimpl/memory src/vespa/storage/frameworkimpl/status src/vespa/storage/frameworkimpl/thread src/vespa/storage/persistence @@ -60,7 +59,6 @@ vespa_define_module( src/tests/common src/tests/common/hostreporter src/tests/distributor - src/tests/frameworkimpl/memory src/tests/frameworkimpl/status src/tests/persistence src/tests/persistence/common diff --git a/storage/src/tests/CMakeLists.txt b/storage/src/tests/CMakeLists.txt index 7747e8bacc1..9962949edde 100644 --- a/storage/src/tests/CMakeLists.txt +++ b/storage/src/tests/CMakeLists.txt @@ -13,7 +13,6 @@ vespa_add_executable(storage_testrunner_app TEST storage_testdistributor storage_testpersistence storage_testfilestorage - storage_testmemory storage_teststatus ) diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index 6906d41ac47..46744ecb3c6 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -24,7 +24,6 @@ #include <vespa/storage/storageserver/framework.h> #include <vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h> #include <vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storageframework/defaultimplementation/component/testcomponentregister.h> #include <vespa/persistence/spi/persistenceprovider.h> diff --git a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt b/storage/src/tests/frameworkimpl/memory/CMakeLists.txt deleted file mode 100644 index e5a364e02a3..00000000000 --- a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testmemory TEST - SOURCES - memorystatusviewertest.cpp - DEPENDS - storage - storage_testcommon -) diff --git a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp b/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp deleted file mode 100644 index 1ad4b16751d..00000000000 --- a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/metrics/metrics.h> -#include <vespa/metrics/metricmanager.h> -#include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h> -#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h> -#include <tests/common/teststorageapp.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/vespalib/util/exceptions.h> -#include <boost/lexical_cast.hpp> - -namespace storage { - -struct MemoryStatusViewerTest : public CppUnit::TestFixture -{ - static const int maxMemory = 1000; - std::unique_ptr<TestServiceLayerApp> _node; - std::unique_ptr<framework::defaultimplementation::MemoryManager> _memMan; - - void setUp() override; - - void testEmptyState(); - void testSnapshots(); - - CPPUNIT_TEST_SUITE(MemoryStatusViewerTest); - CPPUNIT_TEST(testEmptyState); - CPPUNIT_TEST(testSnapshots); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemoryStatusViewerTest); - -void -MemoryStatusViewerTest::setUp() -{ - _node.reset(new TestServiceLayerApp(DiskCount(2))); - framework::defaultimplementation::PriorityMemoryLogic* logic( - new framework::defaultimplementation::PriorityMemoryLogic( - _node->getClock(), maxMemory)); - logic->setMinJumpToUpdateMax(1); - _memMan.reset(new framework::defaultimplementation::MemoryManager( - framework::defaultimplementation::AllocationLogic::UP(logic))); -} - -void -MemoryStatusViewerTest::testEmptyState() -{ - // Add a memory manager, and add a bit of load to it, so it's not - // totally empty. - StorageComponent component(_node->getComponentRegister(), "test"); - - metrics::MetricManager mm; - MemoryStatusViewer viewer( - *_memMan, mm, _node->getComponentRegister()); - std::ostringstream actual; - viewer.reportStatus(actual, framework::HttpUrlPath("/")); - CPPUNIT_ASSERT_MATCH_REGEX(".*Plotr.LineChart.*", actual.str()); - CPPUNIT_ASSERT_MATCH_REGEX( - ".*Current: 1970-01-01 00:00:00 Max memory 1000 SnapShot\\(Used 0, w/o cache 0\\).*", - actual.str()); - CPPUNIT_ASSERT_MATCH_REGEX( - ".*Last hour: na.*", actual.str()); -} - -namespace { - void waitForProcessedTime( - const MemoryStatusViewer& viewer, framework::SecondTime time, - framework::SecondTime timeout = framework::SecondTime(30)) - { - framework::defaultimplementation::RealClock clock; - framework::MilliSecTime endTime( - clock.getTimeInMillis() + timeout.getMillis()); - framework::SecondTime processedTime(0); - while (clock.getTimeInMillis() < endTime) { - processedTime = viewer.getProcessedTime(); - if (processedTime >= time) return; - FastOS_Thread::Sleep(1); - } - std::ostringstream ost; - ost << "Timed out waiting " << timeout << " ms for time " << time - << " to be processed. Currently time is only processed up to " - << processedTime; - throw new vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); - } -} - -#define ASSERT_MEMORY(output, period, maxmem, used, usedwocache) \ -{ \ - std::string::size_type _pos1_(output.find(period)); \ - std::string::size_type _pos2_(output.find("Max memory", _pos1_)); \ - std::string::size_type _pos3_(output.find("SnapShot", _pos2_)); \ - std::string _maxMemory_(output.substr(_pos2_ + 11, _pos3_ - _pos2_ - 12)); \ - std::string::size_type _pos4_(output.find(",", _pos3_)); \ - std::string _used_(output.substr(_pos3_ + 14, _pos4_ - _pos3_ - 14)); \ - std::string::size_type _pos5_(output.find(")", _pos4_)); \ - std::string _usedwo_(output.substr(_pos4_ + 12, _pos5_ - _pos4_ - 12)); \ - std::ostringstream _failure_; \ - _failure_ << "Wrong match in period " << period << " in output:\n" \ - << output << "\nFor value: "; \ - \ - CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Max memory", \ - uint64_t(maxmem), boost::lexical_cast<uint64_t>(_maxMemory_)); \ - CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Used memory", \ - uint64_t(used), boost::lexical_cast<uint64_t>(_used_)); \ - CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Used memory w/o cache", \ - uint64_t(usedwocache), boost::lexical_cast<uint64_t>(_usedwo_)); \ -} - -void -MemoryStatusViewerTest::testSnapshots() -{ - // Add a memory manager, and add a bit of load to it, so it's not - // totally empty. - StorageComponent component(_node->getComponentRegister(), "test"); - const framework::MemoryAllocationType putAlloc( - component.getMemoryManager().registerAllocationType( - framework::MemoryAllocationType("PUT"))); - const framework::MemoryAllocationType getAlloc( - component.getMemoryManager().registerAllocationType( - framework::MemoryAllocationType("GET"))); - - framework::MemoryToken::UP put = _memMan->allocate(putAlloc, 0, 100, 80); - framework::MemoryToken::UP get = _memMan->allocate(getAlloc, 30, 200, 50); - framework::MemoryToken::UP get2 = _memMan->allocate(getAlloc, 70, 150, 60); - - metrics::MetricManager mm; - MemoryStatusViewer viewer(*_memMan, mm, _node->getComponentRegister()); - - _node->getClock().addSecondsToTime(1000); - viewer.notifyThread(); - waitForProcessedTime(viewer, framework::SecondTime(1000)); - - std::ostringstream actual; - viewer.printDebugOutput(actual); - //std::cerr << actual.str() << "\n"; - ASSERT_MEMORY(actual.str(), "Current", 1000, 450, 450); - ASSERT_MEMORY(actual.str(), "Last hour", 1000, 450, 450); - ASSERT_MEMORY(actual.str(), "Last ever", 1000, 450, 450); - - put = _memMan->allocate(putAlloc, 0, 50, 80); - get = _memMan->allocate(getAlloc, 100, 140, 50); - get2 = _memMan->allocate(getAlloc, 20, 100, 70); - - _node->getClock().addSecondsToTime(3600); - viewer.notifyThread(); - waitForProcessedTime(viewer, framework::SecondTime(4600)); - - actual.str(""); - viewer.printDebugOutput(actual); - //std::cerr << actual.str() << "\n"; - ASSERT_MEMORY(actual.str(), "Current", 1000, 290, 290); - ASSERT_MEMORY(actual.str(), "Last hour", 1000, 540, 540); - ASSERT_MEMORY(actual.str(), "Last ever", 1000, 540, 540); - - get.reset(); - - _node->getClock().addSecondsToTime(3600); - viewer.notifyThread(); - waitForProcessedTime(viewer, framework::SecondTime(4600 + 3600)); - - actual.str(""); - viewer.printDebugOutput(actual); - //std::cerr << actual.str() << "\n"; - ASSERT_MEMORY(actual.str(), "Current", 1000, 150, 150); - ASSERT_MEMORY(actual.str(), "Last hour", 1000, 290, 290); - ASSERT_MEMORY(actual.str(), "Last ever", 1000, 540, 540); - -} - -} // storage diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 67b93108139..af0082fa788 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -231,8 +231,6 @@ struct FileStorManagerTest : public CppUnit::TestFixture { fprintf(stderr, "%s\n", e.what()); } _testdoctype1 = _node->getTypeRepo()->getDocumentType("testdoctype1"); - _node->getMemoryManager().registerAllocationType( - framework::MemoryAllocationType("VISITOR_BUFFER")); } void putDoc(DummyStorageLink& top, @@ -2010,19 +2008,11 @@ FileStorManagerTest::testVisiting() top.reset(); // Visit bucket with no split, using no selection { - framework::MemoryToken::UP token( - _node->getMemoryManager().allocate( - _node->getMemoryManager().getAllocationType( - "VISITOR_BUFFER"), - 16*1024, - 16*1024, - 127)); spi::IteratorId iterId(createIterator(top, ids[0], "true")); - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), makeDocumentBucket(ids[0]), iterId, 16*1024)); + auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[0]), iterId, 16*1024); top.sendDown(cmd); top.waitForMessages(1, _waitTime); - CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies()); std::shared_ptr<GetIterReply> reply( std::dynamic_pointer_cast<GetIterReply>(top.getReply(0))); CPPUNIT_ASSERT(reply.get()); @@ -2039,15 +2029,7 @@ FileStorManagerTest::testVisiting() ids[1], "testdoctype1.hstringval = \"John Doe\"")); while (true) { - framework::MemoryToken::UP token( - _node->getMemoryManager().allocate( - _node->getMemoryManager().getAllocationType( - "VISITOR_BUFFER"), - 16*1024, - 16*1024, - 127)); - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), makeDocumentBucket(ids[1]), iterId, 16*1024)); + auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[1]), iterId, 16*1024); top.sendDown(cmd); top.waitForMessages(1, _waitTime); CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); @@ -2078,18 +2060,10 @@ FileStorManagerTest::testVisiting() true)); uint32_t totalDocs = 0; while (true) { - framework::MemoryToken::UP token( - _node->getMemoryManager().allocate( - _node->getMemoryManager().getAllocationType( - "VISITOR_BUFFER"), - 16*1024, - 16*1024, - 127)); - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), makeDocumentBucket(ids[1]), iterId, 16*1024)); + auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[1]), iterId, 16*1024); top.sendDown(cmd); top.waitForMessages(1, _waitTime); - CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies()); std::shared_ptr<GetIterReply> reply( std::dynamic_pointer_cast<GetIterReply>( top.getReply(0))); @@ -2799,15 +2773,8 @@ FileStorManagerTest::testGetIter() // Sending a getiter request that will only visit some of the docs spi::IteratorId iterId(createIterator(top, bid, "")); { - framework::MemoryToken::UP token( - _node->getMemoryManager().allocate( - _node->getMemoryManager().getAllocationType( - "VISITOR_BUFFER"), - 2048, - 2048, - 127)); std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), makeDocumentBucket(bid), iterId, 2048)); + new GetIterCommand(makeDocumentBucket(bid), iterId, 2048)); top.sendDown(cmd); top.waitForMessages(1, _waitTime); CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); @@ -2837,18 +2804,10 @@ FileStorManagerTest::testGetIter() CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); } { - framework::MemoryToken::UP token( - _node->getMemoryManager().allocate( - _node->getMemoryManager().getAllocationType( - "VISITOR_BUFFER"), - 2048, - 2048, - 127)); - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), makeDocumentBucket(bid), iterId, 2048)); + auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(bid), iterId, 2048); top.sendDown(cmd); top.waitForMessages(1, _waitTime); - CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies()); std::shared_ptr<GetIterReply> reply( std::dynamic_pointer_cast<GetIterReply>( top.getReply(0))); diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index fcfbcd0a78b..9d3a3d5f008 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -11,7 +11,6 @@ #include <vespa/vespalib/util/exceptions.h> using document::DocumentType; -using storage::framework::defaultimplementation::AllocationLogic; using storage::spi::test::makeSpiBucket; using document::test::makeDocumentBucket; diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h index ee87925a0bc..36b1fef7285 100644 --- a/storage/src/tests/persistence/persistencetestutils.h +++ b/storage/src/tests/persistence/persistencetestutils.h @@ -8,7 +8,6 @@ #include <vespa/storage/persistence/persistenceutil.h> #include <vespa/storage/common/messagesender.h> #include <vespa/storage/common/storagecomponent.h> -#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/document/base/testdocman.h> diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp index 5111368dd1c..cd5dd2a01a4 100644 --- a/storage/src/tests/storageserver/communicationmanagertest.cpp +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -6,7 +6,6 @@ #include <vespa/messagebus/rpcmessagebus.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> -#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h> #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> #include <tests/common/testhelper.h> diff --git a/storage/src/vespa/storage/CMakeLists.txt b/storage/src/vespa/storage/CMakeLists.txt index 69846cafde2..d854c603097 100644 --- a/storage/src/vespa/storage/CMakeLists.txt +++ b/storage/src/vespa/storage/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_library(storage $<TARGET_OBJECTS:storage_bucketmover> $<TARGET_OBJECTS:storage_thread> $<TARGET_OBJECTS:storage_status> - $<TARGET_OBJECTS:storage_memory> $<TARGET_OBJECTS:storage_component> INSTALL lib64 DEPENDS diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 677ca0a6309..aa9158f9e5b 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -34,7 +34,6 @@ BucketManager::BucketManager(const config::ConfigUri & configUri, : StorageLinkQueued("Bucket manager", compReg), framework::StatusReporter("bucketdb", "Bucket database"), _configUri(configUri), - _bucketDBMemoryToken(), _workerLock(), _workerCond(), _clusterStateLock(), @@ -48,12 +47,6 @@ BucketManager::BucketManager(const config::ConfigUri & configUri, _requestsCurrentlyProcessing(0), _component(compReg, "bucketmanager") { - const framework::MemoryAllocationType& allocType( - _component.getMemoryManager().registerAllocationType( - framework::MemoryAllocationType("DATABASE"))); - _bucketDBMemoryToken = _component.getMemoryManager().allocate( - allocType, 0, 0, api::StorageMessage::HIGH); - assert(_bucketDBMemoryToken.get() != 0); _metrics->setDisks(_component.getDiskCount()); _component.registerStatusPage(*this); _component.registerMetric(*_metrics); @@ -222,8 +215,6 @@ BucketManager::updateMetrics(bool updateDocCount) LOG(debug, "Iterating bucket database to update metrics%s%s", updateDocCount ? "" : ", minusedbits only", _doneInitialized ? "" : ", server is not done initializing"); - uint64_t dbMemSize = _component.getBucketSpaceRepo().getBucketMemoryUsage(); - _bucketDBMemoryToken->resize(dbMemSize, dbMemSize); uint32_t diskCount = _component.getDiskCount(); if (!updateDocCount || _doneInitialized) { diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index 5e4e928dfc7..8413c5a0f42 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -20,7 +20,6 @@ #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/common/storagelinkqueued.h> #include <vespa/storageapi/message/bucket.h> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/storageframework/generic/status/statusreporter.h> @@ -47,7 +46,6 @@ private: config::ConfigUri _configUri; uint32_t _chunkLevel; - framework::MemoryToken::UP _bucketDBMemoryToken; BucketInfoRequestMap _bucketInfoRequests; /** diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp index de44116b316..d458a821e02 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp +++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "statusmetricconsumer.h" -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <boost/assign.hpp> #include <boost/lexical_cast.hpp> @@ -26,17 +25,8 @@ StatusMetricConsumer::StatusMetricConsumer( _component(compReg, "statusmetricsconsumer"), _name(name), _startTime(_component.getClock().getTimeInSeconds()), - _processedTime(0), - _metricMemoryToken() + _processedTime(0) { - const framework::MemoryAllocationType& allocType( - _component.getMemoryManager().registerAllocationType( - framework::MemoryAllocationType( - "METRICS", framework::MemoryAllocationType::FORCE_ALLOCATE) - )); - _metricMemoryToken = _component.getMemoryManager().allocate( - allocType, 0, 0, api::StorageMessage::HIGH); - assert(_metricMemoryToken.get() != 0); LOG(debug, "Started metrics consumer"); setlocale(LC_NUMERIC, ""); _component.registerMetricUpdateHook(*this, framework::SecondTime(3600)); @@ -51,8 +41,7 @@ void StatusMetricConsumer::updateMetrics(const MetricLockGuard & guard) { metrics::MemoryConsumption::UP mc(_manager.getMemoryConsumption(guard)); - uint32_t usage = mc->getTotalMemoryUsage(); - _metricMemoryToken->resize(usage, usage); + // TODO is this hook needed anymore? } vespalib::string diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.h b/storage/src/vespa/storage/common/statusmetricconsumer.h index e3bae51cf99..6f93f51cfdf 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.h +++ b/storage/src/vespa/storage/common/statusmetricconsumer.h @@ -53,7 +53,6 @@ private: vespalib::Monitor _waiter; framework::SecondTime _startTime; framework::SecondTime _processedTime; - std::unique_ptr<framework::MemoryToken> _metricMemoryToken; void writeXmlTags(std::ostream& out, const vespalib::StringTokenizer& name, diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index a3c9804c2b4..29e8d3f6221 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -13,7 +13,6 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/common/storagelink.h> #include <vespa/storageframework/generic/clock/timer.h> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageapi/messageapi/messagehandler.h> #include <set> #include <deque> diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h index e8480902549..8d31a228543 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h @@ -6,7 +6,6 @@ #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/storageapi/messageapi/maintenancecommand.h> #include <vespa/document/bucket/bucketid.h> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> namespace storage::distributor { @@ -229,7 +228,6 @@ protected: bool _ok; api::StorageMessage::Priority _priority; - framework::MemoryToken::UP _memoryToken; /** * Checks if the given bucket is blocked by any pending messages to any diff --git a/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt b/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt deleted file mode 100644 index 2a91ead2b60..00000000000 --- a/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_memory OBJECT - SOURCES - memorystatusviewer.cpp - DEPENDS - AFTER - storage_storageconfig -) diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h b/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h deleted file mode 100644 index fd47bec1424..00000000000 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::MemorySnapshotList - * - * \brief Holds a historic list of MemoryStates. - * - */ - -#pragma once - -#include <storage/frameworkimpl/memory/memorystate.h> - -namespace storage { - -class MemorySnapshotList : public vespalib::Printable { - std::map<uint64_t time, MemoryState> _snapshots; -}; - -} // storage - diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp deleted file mode 100644 index dd07a61f27b..00000000000 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp +++ /dev/null @@ -1,661 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memorystatusviewer.h" -#include <vespa/storage/storageutil/graph.h> -#include <vespa/storage/storageutil/palette.h> -#include <vespa/storage/storageutil/piechart.h> -#include <vespa/metrics/metricmanager.h> -#include <vespa/storageapi/messageapi/storagemessage.h> -#include <algorithm> - -#include <vespa/log/bufferedlogger.h> -LOG_SETUP(".memory.status.viewer"); - -using storage::framework::defaultimplementation::MemoryState; - -namespace storage { - -MemoryStatusViewer::Entry::Entry( - const std::string& name, framework::Clock& clock, - framework::SecondTime maxAge) - : _name(name), - _maxAge(maxAge), - _timeTaken(clock.getTimeInSeconds()), - _data(), - _maxMemory(0) -{ -} - -MemoryStatusViewer::MemoryStatusViewer( - framework::defaultimplementation::MemoryManager& mm, - const metrics::MetricManager& metricMan, - StorageComponentRegister& compReg) - : framework::HtmlStatusReporter("memorymanager", "Memory Manager"), - _component(compReg, "memorystatusviewer"), - _manager(mm), - _metricManager(metricMan), - _workerMonitor(), - _states(), - _memoryHistory(), - _memoryHistorySize(24 * 31), - _memoryHistoryPeriod(60), - _allowedSlackPeriod(6), - _lastHistoryUpdate(_component.getClock().getTimeInSeconds()), - _processedTime(0) -{ - addEntry("Current", 0); - addEntry("Last hour", 60 * 60); - addEntry("Last day", 24 * 60 * 60); - addEntry("Last month", 4 * 7 * 24 * 60 * 60); - addEntry("Last ever", std::numeric_limits<uint32_t>::max()); - - framework::MilliSecTime maxProcessingTime(60 * 1000); - _thread = _component.startThread(*this, maxProcessingTime, - framework::MilliSecTime(1000)); - _component.registerStatusPage(*this); -} - -MemoryStatusViewer::~MemoryStatusViewer() -{ - if (_thread.get() != 0) { - _thread->interrupt(); - { - vespalib::MonitorGuard monitor(_workerMonitor); - monitor.signal(); - } - _thread->join(); - } -} - -namespace { - struct Group { - std::set<const framework::MemoryAllocationType*> types; - api::StorageMessage::Priority minPri; - api::StorageMessage::Priority maxPri; - MemoryState::Entry entry; - - Group(const framework::MemoryAllocationType& type, - api::StorageMessage::Priority pri, - const MemoryState::Entry& e) - : types(), - minPri(pri), - maxPri(pri), - entry(e) - { - types.insert(&type); - } - }; - - struct GroupSizeOrder { - bool operator()(const Group& g1, const Group& g2) { - return (g1.entry._currentUsedSize > g2.entry._currentUsedSize); - } - }; - struct GroupAllocsOrder { - bool operator()(const Group& g1, const Group& g2) { - return (g1.entry._totalUserCount > g2.entry._totalUserCount); - } - }; - struct GroupMinAllocsOrder { - bool operator()(const Group& g1, const Group& g2) { - return (g1.entry._minimumCount > g2.entry._minimumCount); - } - }; - struct GroupDeniedAllocsOrder { - bool operator()(const Group& g1, const Group& g2) { - return (g1.entry._deniedCount > g2.entry._deniedCount); - } - }; - - std::vector<Group> collapsePriorities(MemoryStatusViewer::Entry& entry) { - std::vector<Group> groups; - const MemoryState::SnapShot& ss(entry._data); - for (MemoryState::AllocationMap::const_iterator it - = ss.getAllocations().begin(); - it != ss.getAllocations().end(); ++it) - { - std::unique_ptr<Group> group; - for (MemoryState::PriorityMap::const_iterator it2 - = it->second.begin(); it2 != it->second.end(); ++it2) - { - if (group.get() == 0) { - group.reset(new Group( - *it->first, it2->first, it2->second)); - } else { - group->entry += it2->second; - group->minPri = std::min(group->minPri, it2->first); - group->maxPri = std::max(group->maxPri, it2->first); - } - } - if (group.get() != 0) { - groups.push_back(*group); - } - } - return groups; - } - - std::vector<Group> groupLoad(uint32_t groupCount, uint64_t minSize, - uint32_t minEntries, - MemoryStatusViewer::Entry& entry) - { - assert(groupCount > 1); - std::vector<Group> groups(collapsePriorities(entry)); - if (groups.size() == 0) return groups; - std::sort(groups.begin(), groups.end(), GroupSizeOrder()); - assert(groups.front().entry._currentUsedSize - >= groups.back().entry._currentUsedSize); - while (groups.size() > minEntries - && (groups.size() > groupCount - || groups[groups.size() - 2].entry._currentUsedSize - < minSize)) - { - Group& nextButLast(groups[groups.size() - 2]); - Group& last(groups.back()); - if (last.entry._currentUsedSize > 0) { - nextButLast.entry += last.entry; - nextButLast.minPri = std::min(nextButLast.minPri, last.minPri); - nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri); - nextButLast.types.insert(*last.types.begin()); - } - groups.pop_back(); - } - return groups; - } - - std::vector<Group> groupAllocs(uint32_t groupCount, uint64_t minSize, - uint32_t minEntries, - MemoryStatusViewer::Entry& entry) - { - assert(groupCount > 1); - std::vector<Group> groups(collapsePriorities(entry)); - if (groups.size() == 0) return groups; - std::sort(groups.begin(), groups.end(), GroupAllocsOrder()); - assert(groups.front().entry._totalUserCount - >= groups.back().entry._totalUserCount); - while (groups.size() > minEntries - && (groups.size() > groupCount - || groups[groups.size() - 2].entry._totalUserCount - < minSize)) - { - Group& nextButLast(groups[groups.size() - 2]); - Group& last(groups.back()); - nextButLast.entry += last.entry; - nextButLast.minPri = std::min(nextButLast.minPri, last.minPri); - nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri); - nextButLast.types.insert(*last.types.begin()); - groups.pop_back(); - } - return groups; - } - - std::vector<Group> groupMinAllocs(uint32_t groupCount, uint64_t minSize, - uint32_t minEntries, - MemoryStatusViewer::Entry& entry) - { - assert(groupCount > 1); - std::vector<Group> groups(collapsePriorities(entry)); - if (groups.size() == 0) return groups; - std::sort(groups.begin(), groups.end(), GroupMinAllocsOrder()); - assert(groups.front().entry._minimumCount - >= groups.back().entry._minimumCount); - while (groups.size() > minEntries - && (groups.size() > groupCount - || groups[groups.size() - 2].entry._minimumCount - < minSize)) - { - Group& nextButLast(groups[groups.size() - 2]); - Group& last(groups.back()); - nextButLast.entry += last.entry; - nextButLast.minPri = std::min(nextButLast.minPri, last.minPri); - nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri); - nextButLast.types.insert(*last.types.begin()); - groups.pop_back(); - } - return groups; - } - - std::vector<Group> groupDeniedAllocs(uint32_t groupCount, uint64_t minSize, - uint32_t minEntries, - MemoryStatusViewer::Entry& entry) - { - assert(groupCount > 1); - std::vector<Group> groups(collapsePriorities(entry)); - if (groups.size() == 0) return groups; - std::sort(groups.begin(), groups.end(), GroupDeniedAllocsOrder()); - assert(groups.front().entry._deniedCount - >= groups.back().entry._deniedCount); - while (groups.size() > minEntries - && (groups.size() > groupCount - || groups[groups.size() - 2].entry._deniedCount - < minSize)) - { - Group& nextButLast(groups[groups.size() - 2]); - Group& last(groups.back()); - nextButLast.entry += last.entry; - nextButLast.minPri = std::min(nextButLast.minPri, last.minPri); - nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri); - nextButLast.types.insert(*last.types.begin()); - groups.pop_back(); - } - return groups; - } -} - -void -MemoryStatusViewer::printSnapshot( - std::ostream& out, Entry& entry, - std::map<const framework::MemoryAllocationType*, - uint32_t>& colors) const -{ - out << "<h4>" << entry._name << " - Taken at " - << entry._timeTaken.toString() << "</h4>\n" - << "<table><tr><td>\n" - << "<b>Memory usage"; - if (entry._name != "Current") { - out << ", maxed at " << framework::SecondTime(entry._timeTaken); - } - out << " with " - << (entry._data.getUsedSizeIgnoringCache() / (1024 * 1024)) - << " MB.</b><br>\n"; - std::string piename = entry._name; - std::replace(piename.begin(), piename.end(), ' ', '_'); - uint64_t freeSize = entry._maxMemory - entry._data.getUsedSize(); - // Memory usage pie - uint64_t minSize = freeSize / 20; - std::vector<Group> groups(groupLoad(20, minSize, 5, entry)); - PieChart chart(piename, PieChart::SCHEME_CUSTOM); - chart.printLabels(false); - for (uint32_t i=0; i<groups.size(); ++i) { - std::string name = "Other"; - if (groups[i].types.size() == 1) { - name = (*groups[i].types.begin())->getName(); - } - uint32_t mbytes = groups[i].entry._currentUsedSize / (1024 * 1024); - std::ostringstream ost; - ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri); - if (groups[i].minPri != groups[i].maxPri) { - ost << " - " << static_cast<uint16_t>(groups[i].maxPri); - } - ost << " (" << mbytes << " MB)"; - name = ost.str(); - if (groups[i].entry._currentUsedSize > 0) { - chart.add(groups[i].entry._currentUsedSize, name, - colors[*groups[i].types.begin()]); - } - } - { - std::ostringstream ost; - ost << "Free (" << (freeSize / (1024 * 1024)) << " MB)"; - chart.add(freeSize, ost.str(), colors[0]); - } - chart.printCanvas(out, 750, 300); - out << "\n\n"; - chart.printScript(out, ""); - out << "\n\n"; - // Total allocations pie - out << "</td><td>\n"; - PieChart allocChart(piename + "Alloc", PieChart::SCHEME_CUSTOM); - allocChart.printLabels(false); - groups = groupAllocs(20, 100, 5, entry); - uint64_t totalAllocs = 0; - for (uint32_t i=0; i<groups.size(); ++i) { - std::string name = "Other"; - if (groups[i].types.size() == 1) { - name = (*groups[i].types.begin())->getName(); - } - uint32_t allocs = groups[i].entry._totalUserCount; - totalAllocs += allocs; - std::ostringstream ost; - ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri); - if (groups[i].minPri != groups[i].maxPri) { - ost << " - " << static_cast<uint16_t>(groups[i].maxPri); - } - ost << " (" << allocs << " allocations)"; - name = ost.str(); - if (groups[i].entry._totalUserCount > 0) { - allocChart.add(groups[i].entry._totalUserCount, name, - colors[*groups[i].types.begin()]); - } - } - out << "<b>Allocations, totalling " << totalAllocs << "</b><br>\n"; - allocChart.printCanvas(out, 750, 300); - out << "\n\n"; - allocChart.printScript(out, ""); - out << "\n\n"; - out << "</td></tr><tr><td>\n"; - PieChart minChart(piename + "Min", PieChart::SCHEME_CUSTOM); - minChart.printLabels(false); - groups = groupMinAllocs(20, 100, 5, entry); - uint64_t totalMinAllocs = 0; - for (uint32_t i=0; i<groups.size(); ++i) { - std::string name = "Other"; - if (groups[i].types.size() == 1) { - name = (*groups[i].types.begin())->getName(); - } - uint32_t allocs = groups[i].entry._minimumCount; - totalMinAllocs += allocs; - std::ostringstream ost; - ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri); - if (groups[i].minPri != groups[i].maxPri) { - ost << " - " << static_cast<uint16_t>(groups[i].maxPri); - } - ost << " (" << allocs << " min allocations)"; - name = ost.str(); - if (groups[i].entry._minimumCount > 0) { - minChart.add(groups[i].entry._minimumCount, name, - colors[*groups[i].types.begin()]); - } - } - out << "<b>Minimum allocations, totalling " << totalMinAllocs - << "</b><br>\n"; - if (totalMinAllocs > 0) { - minChart.printCanvas(out, 750, 300); - out << "\n\n"; - minChart.printScript(out, ""); - out << "\n\n"; - } - out << "</td><td>\n"; - PieChart deniedChart(piename + "Denied", PieChart::SCHEME_CUSTOM); - deniedChart.printLabels(false); - groups = groupDeniedAllocs(20, 100, 5, entry); - uint64_t totalDeniedAllocs = 0; - for (uint32_t i=0; i<groups.size(); ++i) { - std::string name = "Other"; - if (groups[i].types.size() == 1) { - name = (*groups[i].types.begin())->getName(); - } - uint32_t allocs = groups[i].entry._deniedCount; - totalDeniedAllocs += allocs; - std::ostringstream ost; - ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri); - if (groups[i].minPri != groups[i].maxPri) { - ost << " - " << static_cast<uint16_t>(groups[i].maxPri); - } - ost << " (" << allocs << " denied allocations)"; - name = ost.str(); - if (groups[i].entry._deniedCount > 0) { - deniedChart.add(groups[i].entry._deniedCount, name, - colors[*groups[i].types.begin()]); - } - } - out << "<b>Denied allocations, totalling " << totalDeniedAllocs - << "</b><br>\n"; - if (totalDeniedAllocs > 0) { - deniedChart.printCanvas(out, 750, 300); - out << "\n\n"; - deniedChart.printScript(out, ""); - out << "\n\n"; - } - out << "</td></tr></table>\n"; -} - -void -MemoryStatusViewer::reportHtmlHeaderAdditions( - std::ostream& out, const framework::HttpUrlPath&) const -{ - (void) out; - // FIXME this function used to emit Yahoo-internal links to graph plotting - // JS files. Obviously, this won't work for external users. Either way, the - // memory manager/status reporter is deprecated. -} - -namespace { - std::map<const framework::MemoryAllocationType*, uint32_t> assignColors( - const std::vector<const framework::MemoryAllocationType*>& types) - { - Palette palette(types.size() + 1); - std::map<const framework::MemoryAllocationType*, uint32_t> colors; - uint32_t nextCol = 0; - colors[0] = palette[nextCol++]; - for (std::vector<const framework::MemoryAllocationType*> - ::const_iterator it = types.begin(); it != types.end(); ++it) - { - colors[*it] = palette[nextCol++]; - } - return colors; - } -} - -void -MemoryStatusViewer::reportHtmlStatus(std::ostream& out, - const framework::HttpUrlPath& path) const -{ - vespalib::MonitorGuard monitor(_workerMonitor); - - if (path.getAttribute("page") == "reset") { - } - if (path.getAttribute("interval") == "current") { - Entry& e(*_states[0]); - out << "<pre>" << e._name << ": "; - if (e.containsData()) { - e._data.print(out, true, " "); - } else { - out << "na"; - } - out << "\n</pre>"; - return; - } - const_cast<MemoryStatusViewer*>(this)->grabMemoryUsage(); - framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); - std::vector<const framework::MemoryAllocationType*> allocTypes( - _manager.getAllocationTypes()); - std::map<const framework::MemoryAllocationType*, uint32_t> colors( - assignColors(allocTypes)); - // Print memory usage graph - { - uint32_t mb = 1024 * 1024; - Graph memoryHistory("memhistory", Graph::SCHEME_CUSTOM); - std::vector<Graph::Point> total; - std::vector<Graph::Point> used; - std::vector<Graph::Point> usedWoCache; - uint32_t xval = 0; - for (std::deque<MemoryTimeEntry>::const_iterator it - = _memoryHistory.begin(); it != _memoryHistory.end(); - ++it, ++xval) - { - used.push_back(Graph::Point(xval, it->used)); - usedWoCache.push_back(Graph::Point(xval, it->usedWithoutCache)); - } - used.push_back(Graph::Point( - xval, _states[0]->_data.getUsedSize() / mb)); - usedWoCache.push_back(Graph::Point( - xval, _states[0]->_data.getUsedSizeIgnoringCache() / mb)); - uint32_t totalSize = _states[0]->_maxMemory / mb; - total.push_back(Graph::Point(0, totalSize)); - total.push_back(Graph::Point(xval, totalSize)); - memoryHistory.add(total, "Total memory", Graph::GREEN); - memoryHistory.add(used, "Used memory", Graph::YELLOW); - memoryHistory.add(usedWoCache, "Used memory excluding freeable cache", - Graph::RED); - out << "<p>Memory available for lowest priority (255): " - << _manager.getMemorySizeFreeForPriority(255) << " byte(s).</p>\n"; - out << "<h3>Historic memory usage</h3>\n"; - uint32_t yAxisUnit = ((totalSize / 4) / 256) * 256; - if (yAxisUnit == 0) yAxisUnit = (totalSize / 4); - if (yAxisUnit == 0) yAxisUnit = 1; - uint32_t size = yAxisUnit; - memoryHistory.addYAxisLabel(0, "0 B"); - while (size <= totalSize) { - std::ostringstream label; - if (size % 1024 == 0) { - label << (size / 1024) << " GB"; - } else { - label << size << " MB"; - } - memoryHistory.addYAxisLabel(size, label.str()); - size += yAxisUnit; - } - uint32_t xAxisUnit = ((_memoryHistory.size() / 4) / 24) * 24; - if (xAxisUnit == 0) xAxisUnit = _memoryHistoryPeriod.getTime(); - uint32_t startTime = ((currentTime.getTime() - / _memoryHistoryPeriod.getTime()) - / 24) * 24; - uint32_t stopTime = (currentTime.getTime() - / _memoryHistoryPeriod.getTime()) - - _memoryHistory.size() + 1; - memoryHistory.addXAxisLabel(xval, currentTime.toString()); - bool addedMiddlePoints = false; - while (startTime >= stopTime) { - if (currentTime.getTime() / _memoryHistoryPeriod.getTime() - - startTime > 48) - { - memoryHistory.addXAxisLabel( - (startTime - stopTime), - framework::SecondTime( - startTime * _memoryHistoryPeriod.getTime()) - .toString()); - addedMiddlePoints = true; - } - startTime -= xAxisUnit; - } - if (!addedMiddlePoints && _memoryHistory.size() > 2) { - memoryHistory.addXAxisLabel( - 1, - framework::SecondTime( - stopTime * _memoryHistoryPeriod.getTime()) - .toString()); - } - memoryHistory.setBorders(50, 0, 0, 30); - memoryHistory.setLegendPos(80, 20); - memoryHistory.printCanvas(out, 1000, 250); - memoryHistory.printScript(out, ""); - } - uint32_t maxUsedWithoutCache = 0; - for (uint32_t i=0; i<_states.size(); ++i) { - Entry& e(*_states[i]); - if (!e.containsData() - || e._data.getUsedSizeIgnoringCache() == maxUsedWithoutCache) - { - continue; - } - printSnapshot(out, e, colors); - maxUsedWithoutCache = e._data.getUsedSizeIgnoringCache(); - } - out << "<h3>Raw output of stored data</h3>\n" - << "<pre>\n"; - monitor.unlock(); - printDebugOutput(out); - out << "</pre>\n"; - out << "<h2>Memory used for metrics. (Not tracked in memory manager)</h2>\n" - << "<pre>\n" - << _metricManager.getMemoryConsumption(_metricManager.getMetricLock())->toString() - << "\n</pre>\n"; -} - -void -MemoryStatusViewer::run(framework::ThreadHandle& thread) -{ - while (!thread.interrupted()) { - vespalib::MonitorGuard monitor(_workerMonitor); - framework::SecondTime currentTime( - _component.getClock().getTimeInSeconds()); - if (_lastHistoryUpdate + _memoryHistoryPeriod <= currentTime - || _states[0]->_timeTaken + _memoryHistoryPeriod <= currentTime) - { - grabMemoryUsage(); - _processedTime = currentTime; - LOG(spam, "Done processing time %" PRIu64, currentTime.getTime()); - thread.registerTick(framework::PROCESS_CYCLE); - } else { - monitor.wait(thread.getWaitTime()); - thread.registerTick(framework::WAIT_CYCLE); - } - } -} - -// You should have worker monitor when calling this function -void -MemoryStatusViewer::grabMemoryUsage() -{ - framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); - MemoryState state(_component.getClock(), 0); - _manager.getState(state, true); - - if (_lastHistoryUpdate + _memoryHistoryPeriod <= currentTime) { - LOG(spam, "Adding another %" PRIu64 " sec entry to memory history.", - _memoryHistoryPeriod.getTime()); - // Add history once an hour - uint32_t mb = 1024 * 1024; - _memoryHistory.push_back(MemoryTimeEntry( - state.getMaxSnapshot().getUsedSize() / mb, - state.getMaxSnapshot().getUsedSizeIgnoringCache() / mb)); - if (_memoryHistory.size() > _memoryHistorySize) { - if (_memoryHistoryPeriod != framework::SecondTime(60 * 60)) { - uint32_t periodDiff = 60 * 60 / _memoryHistoryPeriod.getTime(); - std::deque<MemoryTimeEntry> newHistory; - uint32_t count = 0; - MemoryTimeEntry entry(0, 0); - for (std::deque<MemoryTimeEntry>::const_iterator it - = _memoryHistory.begin(); - it != _memoryHistory.end(); ++it) - { - entry.keepMax(*it); - if (++count == periodDiff) { - newHistory.push_back(entry); - entry = MemoryTimeEntry(0, 0); - count = 0; - } - } - if (entry.used != 0) { - newHistory.push_back(entry); - } - _memoryHistory.swap(newHistory); - _memoryHistoryPeriod = framework::SecondTime(60 * 60); - } - } - _lastHistoryUpdate += _memoryHistoryPeriod; - if (_lastHistoryUpdate + _allowedSlackPeriod < currentTime) { - LOGBP(warning, "Memory history is supposed to be tracked every %" - PRIu64 " seconds, but %" PRIu64" seconds have passed " - "since last update. Memory history graph will be " - "incorrect.", - _memoryHistoryPeriod.getTime(), - (currentTime - _lastHistoryUpdate + _memoryHistoryPeriod) - .getTime()); - _lastHistoryUpdate = currentTime; - } - } - LOG(spam, "Overwriting current with snapshot using %" PRIu64 " bytes.", - state.getCurrentSnapshot().getUsedSize()); - _states[0]->assign(state.getCurrentSnapshot(), - state.getTotalSize(), currentTime); - for (uint32_t i=1, n=_states.size(); i<n; ++i) { - if (currentTime - _states[i]->_timeTaken >= _states[i]->_maxAge - || state.getMaxSnapshot().getUsedSize() - > _states[i]->_data.getUsedSize()) - { - LOG(spam, "Updating period %s usage. Old usage was %" PRIu64 ". " - "Last set at %" PRIu64, - _states[i]->_name.c_str(), _states[i]->_data.getUsedSize(), - _states[i]->_timeTaken.getTime()); - _states[i]->assign(state.getMaxSnapshot(), - state.getTotalSize(), currentTime); - } - } -} - -void -MemoryStatusViewer::notifyThread() const -{ - vespalib::MonitorGuard monitor(_workerMonitor); - monitor.broadcast(); -} - -void -MemoryStatusViewer::printDebugOutput(std::ostream& out) const -{ - vespalib::MonitorGuard monitor(_workerMonitor); - for (uint32_t i=0; i<_states.size(); ++i) { - Entry& e(*_states[i]); - out << e._name << ": "; - if (e.containsData()) { - out << e._timeTaken.toString() << " Max memory " << e._maxMemory << " "; - e._data.print(out, true, " "); - } else { - out << "na"; - } - out << "\n\n"; - } -} - -} // storage diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h deleted file mode 100644 index 2c7510d371d..00000000000 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::MemoryStatusViewer - * - * \brief Generates status to access through status pages. - * - * Keeps a history of the largest memory inprints seen historically. This is - * done be defining periods, where a period is always a multiplum of the length - * of the period shorter than it. The last entry will store the biggest memory - * imprint ever seen, and the earlier entries will show biggest for their time - * period. - * - * To avoid having all periods cleared once the biggest period resets, the - * periods keep data for each of the periods one size below it. Thus, a year - * keeps data for 12 months, a month for 30 days, and so on. - * - * The memory state objects are divided in 3 parts. Current memory data, max - * memory data since since reset and counts for how often various events have - * happened. - * - * The counts will have their total count values stored in the current entry. - * When the next period is updated getting a copy of these counts, we can see - * how many counts have happened recently, by taking the current entry and - * subtract those accounted for earlier. - * - * The current memory data will not be interesting for anything than to show the - * actual now values in the current entry. - * - * The max since reset values will be the values used for the various periods. - * When a period is updated with new data for a subpart of their period, the - * max seen data is reset in the period in front, such that a lower maximum - * can be found. - */ - -#pragma once - -#include <vespa/storage/common/storagecomponent.h> -#include <vespa/storageframework/defaultimplementation/memory/memorystate.h> -#include <vespa/storageframework/generic/status/htmlstatusreporter.h> -#include <vespa/vespalib/util/document_runnable.h> -#include <vespa/vespalib/util/sync.h> -#include <deque> -#include <vector> - - -namespace metrics { - class MetricManager; -} - -namespace storage { - -class StorageServerInterface; - -class MemoryStatusViewer : public framework::HtmlStatusReporter, - private framework::Runnable -{ -public: - typedef framework::defaultimplementation::MemoryState::SnapShot SnapShot; - struct Entry { - typedef std::shared_ptr<Entry> SP; - - std::string _name; - framework::SecondTime _maxAge; - framework::SecondTime _timeTaken; - SnapShot _data; - uint64_t _maxMemory; - - Entry(const std::string& name, framework::Clock&, - framework::SecondTime maxAge); - bool containsData() const { return (_maxMemory != 0); } - - void assign(const SnapShot& snapshot, uint64_t maxMemory, - framework::SecondTime time) - { - _data = snapshot; - _maxMemory = maxMemory; - _timeTaken = time; - } - }; - - struct MemoryTimeEntry { - uint64_t used; - uint64_t usedWithoutCache; - - MemoryTimeEntry(uint64_t u, uint64_t wo) - : used(u), usedWithoutCache(wo) {} - - void keepMax(const MemoryTimeEntry& e) { - used = (used > e.used ? used : e.used); - usedWithoutCache = (usedWithoutCache > e.usedWithoutCache - ? usedWithoutCache : e.usedWithoutCache); - } - }; - -private: - framework::Component _component; - framework::defaultimplementation::MemoryManager& _manager; - const metrics::MetricManager& _metricManager; - vespalib::Monitor _workerMonitor; - - std::vector<Entry::SP> _states; - std::deque<MemoryTimeEntry> _memoryHistory; - uint32_t _memoryHistorySize; - framework::SecondTime _memoryHistoryPeriod; - framework::SecondTime _allowedSlackPeriod; - framework::SecondTime _lastHistoryUpdate; - framework::Thread::UP _thread; - framework::SecondTime _processedTime; - - void addEntry(const std::string& name, uint32_t maxAge) { - _states.push_back(Entry::SP(new Entry(name, _component.getClock(), - framework::SecondTime(maxAge)))); - } - void run(framework::ThreadHandle&) override; - void grabMemoryUsage(); - void printSnapshot(std::ostream& out, Entry& entry, - std::map<const framework::MemoryAllocationType*, - uint32_t>& colors) const; - -public: - MemoryStatusViewer( - framework::defaultimplementation::MemoryManager&, - const metrics::MetricManager&, - StorageComponentRegister&); - ~MemoryStatusViewer(); - - void reportHtmlHeaderAdditions(std::ostream&, const framework::HttpUrlPath&) const override; - void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; - - /** Useful for testing. */ - framework::SecondTime getProcessedTime() const { return _processedTime; } - void notifyThread() const; - void printDebugOutput(std::ostream&) const; - -}; - -} diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp index 876067e4e15..720371a68f1 100644 --- a/storage/src/vespa/storage/persistence/messages.cpp +++ b/storage/src/vespa/storage/persistence/messages.cpp @@ -6,17 +6,14 @@ using document::BucketSpace; namespace storage { -GetIterCommand::GetIterCommand(framework::MemoryToken::UP token, - const document::Bucket &bucket, +GetIterCommand::GetIterCommand(const document::Bucket &bucket, const spi::IteratorId iteratorId, uint32_t maxByteSize) : api::InternalCommand(ID), - _token(std::move(token)), _bucket(bucket), _iteratorId(iteratorId), _maxByteSize(maxByteSize) { - assert(_token.get()); } GetIterCommand::~GetIterCommand() { } @@ -38,7 +35,6 @@ GetIterCommand::makeReply() { GetIterReply::GetIterReply(GetIterCommand& cmd) : api::InternalReply(ID, cmd), - _token(cmd.releaseMemoryToken()), _bucket(cmd.getBucket()), _completed(false) { } diff --git a/storage/src/vespa/storage/persistence/messages.h b/storage/src/vespa/storage/persistence/messages.h index 7bbeea8a12a..ba7f5979569 100644 --- a/storage/src/vespa/storage/persistence/messages.h +++ b/storage/src/vespa/storage/persistence/messages.h @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storageframework/generic/memory/memorytoken.h> #include <vespa/storageapi/message/internal.h> #include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/bucket.h> @@ -14,7 +13,6 @@ namespace storage { class GetIterCommand : public api::InternalCommand { private: - mutable framework::MemoryToken::UP _token; document::Bucket _bucket; spi::IteratorId _iteratorId; uint32_t _maxByteSize; @@ -24,8 +22,7 @@ public: typedef std::unique_ptr<GetIterCommand> UP; typedef std::shared_ptr<GetIterCommand> SP; - GetIterCommand(framework::MemoryToken::UP token, - const document::Bucket &bucket, + GetIterCommand(const document::Bucket &bucket, const spi::IteratorId iteratorId, uint32_t maxByteSize); ~GetIterCommand(); @@ -44,13 +41,11 @@ public: void print(std::ostream& out, bool verbose, const std::string& indent) const override; private: - framework::MemoryToken::UP releaseMemoryToken() { return std::move(_token); } friend class GetIterReply; }; class GetIterReply : public api::InternalReply { private: - framework::MemoryToken::UP _token; document::Bucket _bucket; std::vector<spi::DocEntry::UP> _entries; bool _completed; diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 3d0a0fdfa14..c0238922a91 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -11,7 +11,6 @@ vespa_add_library(storage_storageserver documentapiconverter.cpp fnetlistener.cpp mergethrottler.cpp - messageallocationtypes.cpp messagesink.cpp opslogger.cpp priorityconverter.cpp diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index eae51b90165..b214ef4539a 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -96,13 +96,6 @@ StorageTransportContext::StorageTransportContext(std::unique_ptr<RPCRequestWrapp StorageTransportContext::~StorageTransportContext() { } -const framework::MemoryAllocationType& -CommunicationManager::getAllocationType(api::StorageMessage& msg) const -{ - return _messageAllocTypes.getType(msg.getType().getId()); -} - - void CommunicationManager::receiveStorageReply(const std::shared_ptr<api::StorageReply>& reply) { @@ -296,8 +289,7 @@ CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, co _configUri(configUri), _closed(false), _bucketResolver(std::make_unique<PlaceHolderBucketResolver>()), - _docApiConverter(configUri, *_bucketResolver), - _messageAllocTypes(_component.getMemoryManager()) + _docApiConverter(configUri, *_bucketResolver) { _component.registerMetricUpdateHook(*this, framework::SecondTime(5)); _component.registerMetric(_metrics); @@ -483,33 +475,9 @@ CommunicationManager::process(const std::shared_ptr<api::StorageMessage>& msg) void CommunicationManager::enqueue(const std::shared_ptr<api::StorageMessage> & msg) { - using MemoryToken = framework::MemoryToken; assert(msg.get()); - - const uint32_t memoryFootprint = msg->getMemoryFootprint(); - MemoryToken::UP token = _component.getMemoryManager().allocate(getAllocationType(*msg), memoryFootprint * 2, - memoryFootprint * 2, msg->getPriority()); - - if (token) { - msg->setMemoryToken(std::move(token)); - - LOG(spam, "Enq storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); - _eventQueue.enqueue(msg); - } else { - _metrics.failedDueToTooLittleMemory.inc(); - std::ostringstream ost; - ost << "Failed to aquire " << (memoryFootprint * 2) - << " bytes of memory to handle command of type " - << msg->getType() << "\n"; - LOG(spam, "%s", ost.str().c_str()); - api::StorageCommand* cmd(dynamic_cast<api::StorageCommand*>(msg.get())); - - if (cmd) { - std::shared_ptr<api::StorageReply> reply(cmd->makeReply()); - reply->setResult(api::ReturnCode(api::ReturnCode::BUSY, ost.str())); - sendReply(reply); - } - } + LOG(spam, "Enq storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); + _eventQueue.enqueue(msg); } bool diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index 4cf3f33e6ea..89b76583cb5 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -11,7 +11,6 @@ #pragma once #include "communicationmanagermetrics.h" -#include "messageallocationtypes.h" #include "documentapiconverter.h" #include <vespa/storage/common/storagelink.h> #include <vespa/storage/common/storagecomponent.h> @@ -171,10 +170,7 @@ private: std::unique_ptr<BucketResolver> _bucketResolver; DocumentApiConverter _docApiConverter; framework::Thread::UP _thread; - MessageAllocationTypes _messageAllocTypes; - const framework::MemoryAllocationType& - getAllocationType(api::StorageMessage& msg) const; void updateMetrics(const MetricLockGuard &) override; // Test needs access to configure() for live reconfig testing. diff --git a/storage/src/vespa/storage/storageserver/framework.h b/storage/src/vespa/storage/storageserver/framework.h index 724b4fe0944..f0ea1d71aa6 100644 --- a/storage/src/vespa/storage/storageserver/framework.h +++ b/storage/src/vespa/storage/storageserver/framework.h @@ -18,7 +18,6 @@ #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> -#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> namespace storage { @@ -27,7 +26,6 @@ struct Framework { // Typedefs to simplify the remainder of the interface typedef StorageComponentRegisterImpl CompReg; typedef framework::defaultimplementation::RealClock RealClock; - typedef framework::defaultimplementation::MemoryManager MemoryManager; /** * You can provide your own clock implementation. Useful in testing where @@ -48,20 +46,10 @@ struct Framework { */ FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); } - /** - * Get the memory manager. Components that wants to print status of memory - * manager need access to the actual implementation. - */ - MemoryManager& getMemoryManager() { return _memoryManager; } - - void setMaximumMemoryUsage(uint64_t max); - private: CompReg _componentRegister; framework::Clock::UP _clock; framework::defaultimplementation::ThreadPoolImpl _threadPool; - framework::defaultimplementation::AllocationLogic* _memoryLogic; - MemoryManager _memoryManager; }; diff --git a/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp b/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp deleted file mode 100644 index ac2d99a439c..00000000000 --- a/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "messageallocationtypes.h" -#include <vespa/storageapi/messageapi/storagemessage.h> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/stllike/asciistream.h> - -namespace storage { - -MessageAllocationTypes::MessageAllocationTypes(framework::MemoryManagerInterface& manager) -{ - using api::MessageType; - using framework::MemoryAllocationType; - - _types.resize(MessageType::MESSAGETYPE_MAX_ID); - _types[MessageType::DOCBLOCK_ID] = &manager.registerAllocationType(MemoryAllocationType("MESSAGE_DOCBLOCK")); - _types[MessageType::DOCBLOCK_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MESSAGE_DOCBLOCK_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GET", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::GET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::INTERNAL_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::INTERNAL")); - _types[MessageType::INTERNAL_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::INTERNAL_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::PUT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::PUT", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::PUT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::PUT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::REMOVE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVE", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::REMOVE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::REVERT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REVERT")); - _types[MessageType::REVERT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REVERT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::VISITOR_CREATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_CREATE", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::VISITOR_CREATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_CREATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::VISITOR_DESTROY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_DESTROY")); - _types[MessageType::VISITOR_DESTROY_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_DESTROY_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::REQUESTBUCKETINFO_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REQUESTBUCKETINFO")); - _types[MessageType::REQUESTBUCKETINFO_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REQUESTBUCKETINFO_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::NOTIFYBUCKETCHANGE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::NOTIFYBUCKETCHANGE")); - _types[MessageType::NOTIFYBUCKETCHANGE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::NOTIFYBUCKETCHANGE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::CREATEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::CREATEBUCKET")); - _types[MessageType::CREATEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::CREATEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::MERGEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MERGEBUCKET")); - _types[MessageType::MERGEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MERGEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::DELETEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DELETEBUCKET")); - _types[MessageType::DELETEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DELETEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SETNODESTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETNODESTATE", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SETNODESTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETNODESTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETNODESTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETNODESTATE", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETNODESTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETNODESTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SETSYSTEMSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETSYSTEMSTATE", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SETSYSTEMSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETSYSTEMSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETSYSTEMSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETSYSTEMSTATE", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETSYSTEMSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETSYSTEMSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETBUCKETDIFF_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETDIFF", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETBUCKETDIFF_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETDIFF_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::APPLYBUCKETDIFF_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::APPLYBUCKETDIFF", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::APPLYBUCKETDIFF_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::APPLYBUCKETDIFF_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::VISITOR_INFO_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_INFO")); - _types[MessageType::VISITOR_INFO_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_INFO_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SEARCHRESULT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SEARCHRESULT")); - _types[MessageType::SEARCHRESULT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SEARCHRESULT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SPLITBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SPLITBUCKET")); - _types[MessageType::SPLITBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SPLITBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::JOINBUCKETS_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::JOINBUCKETS")); - _types[MessageType::JOINBUCKETS_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::JOINBUCKETS_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SETBUCKETSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETBUCKETSTATE")); - _types[MessageType::SETBUCKETSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETBUCKETSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::MULTIOPERATION_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MULTIOPERATION", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::MULTIOPERATION_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MULTIOPERATION_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::DOCUMENTSUMMARY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTSUMMARY")); - _types[MessageType::DOCUMENTSUMMARY_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTSUMMARY_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::MAPVISITOR_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MAPVISITOR")); - _types[MessageType::MAPVISITOR_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MAPVISITOR_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::STATBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::STATBUCKET", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::STATBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::STATBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETBUCKETLIST_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETLIST", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::GETBUCKETLIST_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETLIST_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::DOCUMENTLIST_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTLIST")); - _types[MessageType::DOCUMENTLIST_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTLIST_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::UPDATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::UPDATE", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::UPDATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::UPDATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::EMPTYBUCKETS_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::EMPTYBUCKETS")); - _types[MessageType::EMPTYBUCKETS_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::EMPTYBUCKETS_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::REMOVELOCATION_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVELOCATION", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::REMOVELOCATION_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVELOCATION_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::QUERYRESULT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::QUERYRESULT")); - _types[MessageType::QUERYRESULT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::QUERYRESULT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::BATCHPUTREMOVE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHPUTREMOVE", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::BATCHPUTREMOVE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHPUTREMOVE_REPLY", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::BATCHDOCUMENTUPDATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHDOCUMENTUPDATE", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::BATCHDOCUMENTUPDATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHDOCUMENTUPDATE_REPLY", framework::MemoryAllocationType::EXTERNAL_LOAD)); -} - -const framework::MemoryAllocationType& -MessageAllocationTypes::getType(uint32_t type) const { - if (_types.size() > size_t(type) && _types[type] != 0) { - return *_types[type]; - } - vespalib::asciistream ost; - ost << "No type registered with value " << type << "."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); -} - -} // storage diff --git a/storage/src/vespa/storage/storageserver/messageallocationtypes.h b/storage/src/vespa/storage/storageserver/messageallocationtypes.h deleted file mode 100644 index 8f7e8e4b3c3..00000000000 --- a/storage/src/vespa/storage/storageserver/messageallocationtypes.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::MessageAllocationTypes - * - * \brief Memory allocation types for messages in storage. - */ -#pragma once - -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> -#include <vector> - -namespace storage { - -class MessageAllocationTypes { - std::vector<const framework::MemoryAllocationType*> _types; - -public: - MessageAllocationTypes(framework::MemoryManagerInterface& manager); - - const framework::MemoryAllocationType& getType(uint32_t type) const; -}; - -} - diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 855efaed6aa..ba1556bd3b9 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -7,7 +7,6 @@ #include "storagemetricsset.h" #include "storagenodecontext.h" -#include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h> #include <vespa/storage/frameworkimpl/status/statuswebserver.h> #include <vespa/storage/frameworkimpl/thread/deadlockdetector.h> #include <vespa/storage/common/statusmetricconsumer.h> @@ -117,7 +116,6 @@ StorageNode::initialize() // and store them away, while having the config lock. subscribeToConfigs(); - _context.getMemoryManager().setMaximumMemoryUsage(_serverConfig->memorytouse); updateUpgradeFlag(*_clusterConfig); // First update some basics that doesn't depend on anything else to be @@ -157,11 +155,6 @@ StorageNode::initialize() initializeNodeSpecific(); - _memoryStatusViewer.reset(new MemoryStatusViewer( - _context.getMemoryManager(), - _context.getComponentRegister().getMetricManager(), - _context.getComponentRegister())); - _statusMetrics.reset(new StatusMetricConsumer( _context.getComponentRegister(), _context.getComponentRegister().getMetricManager())); _stateReporter.reset(new StateReporter( @@ -259,14 +252,6 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) DIFFERWARN(clusterName, "Cannot alter cluster name of node live"); DIFFERWARN(nodeIndex, "Cannot alter node index of node live"); DIFFERWARN(isDistributor, "Cannot alter role of node live"); - { - if (DIFFER(memorytouse)) { - LOG(info, "Live config update: Memory to use changed from %" PRId64 " to %" PRId64 ".", - oldC.memorytouse, newC.memorytouse); - ASSIGN(memorytouse); - _context.getMemoryManager().setMaximumMemoryUsage(newC.memorytouse); - } - } _serverConfig.reset(new StorServerConfig(oldC)); _newServerConfig.reset(); (void)updated; @@ -413,10 +398,6 @@ StorageNode::shutdown() LOG(debug, "Deleting state reporter"); _stateReporter.reset(); } - if (_memoryStatusViewer) { - LOG(debug, "Deleting memory status viewer"); - _memoryStatusViewer.reset(); - } if (_stateManager) { LOG(debug, "Deleting state manager"); _stateManager.reset(); diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index 9b727ef3e0c..e9d3004be68 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -121,8 +121,6 @@ private: // Depends on bucket databases and stop() functionality std::unique_ptr<DeadLockDetector> _deadLockDetector; - // Depends on dead lock detector and threadpool - std::unique_ptr<MemoryStatusViewer> _memoryStatusViewer; // Depends on metric manager std::unique_ptr<StatusMetricConsumer> _statusMetrics; // Depends on metric manager diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp index 2e1aa52e68d..75e1f12773f 100644 --- a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp +++ b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp @@ -2,31 +2,15 @@ #include "storagenodecontext.h" -#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h> - namespace storage { -using framework::defaultimplementation::AllocationLogic; -using framework::defaultimplementation::PriorityMemoryLogic; - StorageNodeContext::StorageNodeContext(ComponentRegister::UP compReg, framework::Clock::UP clock) : _componentRegister(std::move(compReg)), _clock(std::move(clock)), - _threadPool(*_clock), - _memoryLogic(new PriorityMemoryLogic(*_clock, 1024 * 1024 * 1024)), - _memoryManager(AllocationLogic::UP(_memoryLogic)) + _threadPool(*_clock) { _componentRegister->setClock(*_clock); _componentRegister->setThreadPool(_threadPool); - _componentRegister->setMemoryManager(_memoryManager); -} - -void -StorageNodeContext::setMaximumMemoryUsage(uint64_t max) -{ - using storage::framework::defaultimplementation::PriorityMemoryLogic; - dynamic_cast<PriorityMemoryLogic*>(_memoryLogic) - ->setMaximumMemoryUsage(max); } } // storage diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.h b/storage/src/vespa/storage/storageserver/storagenodecontext.h index 0149f975f63..eabca618bfb 100644 --- a/storage/src/vespa/storage/storageserver/storagenodecontext.h +++ b/storage/src/vespa/storage/storageserver/storagenodecontext.h @@ -18,7 +18,6 @@ #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> -#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> namespace storage { @@ -27,7 +26,6 @@ struct StorageNodeContext { // Typedefs to simplify the remainder of the interface typedef StorageComponentRegisterImpl ComponentRegister; typedef framework::defaultimplementation::RealClock RealClock; - typedef framework::defaultimplementation::MemoryManager MemoryManager; /** * Get the actual component register. Available as the actual type as the @@ -42,14 +40,6 @@ struct StorageNodeContext { */ FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); } - /** - * Get the memory manager. Components that wants to print status of memory - * manager need access to the actual implementation. - */ - MemoryManager& getMemoryManager() { return _memoryManager; } - - void setMaximumMemoryUsage(uint64_t max); - protected: // Initialization has been split in two as subclass needs to initialize // component register before sending it on. @@ -59,8 +49,6 @@ private: ComponentRegister::UP _componentRegister; framework::Clock::UP _clock; framework::defaultimplementation::ThreadPoolImpl _threadPool; - framework::defaultimplementation::AllocationLogic* _memoryLogic; - MemoryManager _memoryManager; }; diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index b91e38e035e..0d230745875 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -2,7 +2,6 @@ #include "visitor.h" #include "visitormetrics.h" -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storage/persistence/messages.h> @@ -267,8 +266,7 @@ Visitor::Visitor(StorageComponent& component) _id(), _controlDestination(), _dataDestination(), - _documentSelection(), - _memoryManager(0) + _documentSelection() { } @@ -603,10 +601,6 @@ Visitor::start(api::VisitorId id, api::StorageMessage::Id cmdId, _documentPriority = documentPriority; _state = STATE_RUNNING; - if (_memoryAllocType == 0) { - _memoryAllocType = &_component.getMemoryManager() - .getAllocationType("VISITOR_BUFFER"); - } LOG(debug, "Starting visitor '%s' for %" PRIu64 " buckets from %" PRIu64 " to " "%" PRIu64 ". First is %s. Max pending replies: %u, include " @@ -803,17 +797,7 @@ Visitor::onCreateIteratorReply( LOG(debug, "Visitor '%s' starting to visit bucket %s.", _id.c_str(), bucketId.toString().c_str()); - framework::MemoryToken::UP token( - _memoryManager->allocate( - *_memoryAllocType, _docBlockSize, _docBlockSize, _priority)); - if (token.get() == 0) { - // Not enough memory - return; - } - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), bucket, - bucketState.getIteratorId(), - _docBlockSize)); + auto cmd = std::make_shared<GetIterCommand>(bucket, bucketState.getIteratorId(), _docBlockSize); cmd->setLoadType(_initiatingCmd->getLoadType()); cmd->getTrace().setLevel(_traceLevel); cmd->setPriority(_priority); @@ -1223,19 +1207,8 @@ Visitor::getIterators() it = _bucketStates.erase(it); continue; } - framework::MemoryToken::UP token( - _memoryManager->allocate( - *_memoryAllocType, _docBlockSize, _docBlockSize, - _priority)); - if (token.get() == 0) { - // Not enough memory - return true; - } - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), - bucketState.getBucket(), - bucketState.getIteratorId(), - _docBlockSize)); + auto cmd = std::make_shared<GetIterCommand>( + bucketState.getBucket(), bucketState.getIteratorId(), _docBlockSize); cmd->setLoadType(_initiatingCmd->getLoadType()); cmd->getTrace().setLevel(_traceLevel); cmd->setPriority(_priority); diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 4436312032f..f84b105f1ed 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -327,7 +327,6 @@ private: protected: // These variables should not be altered after visitor starts. This not // controlled by locks. - const framework::MemoryAllocationType* _memoryAllocType; VisitorMessageHandler* _messageHandler; VisitorMessageSession::UP _messageSession; documentapi::Priority::Value _documentPriority; @@ -339,7 +338,6 @@ protected: std::string _documentSelectionString; std::unique_ptr<document::OrderingSpecification> _ordering; vdslib::VisitorStatistics _visitorStatistics; - framework::MemoryManagerInterface* _memoryManager; bool isCompletedCalled() const { return _calledCompletedVisitor; } @@ -365,8 +363,6 @@ public: const api::StorageMessageAddress* getDataDestination() const { return _dataDestination.get(); } // Can't be null if attached - void setAllocationType(const framework::MemoryAllocationType& mat) - { _memoryAllocType = &mat; } void setMaxPending(unsigned int maxPending) { _visitorOptions._maxPending = maxPending; } @@ -384,8 +380,6 @@ public: { _docBlockTimeout = timeout; } void setVisitorInfoTimeout(framework::MilliSecTime timeout) { _visitorInfoTimeout = timeout; } - void setMemoryManager(framework::MemoryManagerInterface& mm) - { _memoryManager = &mm; } void setOwnNodeIndex(uint16_t nodeIndex) { _ownNodeIndex = nodeIndex; } void setBucketSpace(document::BucketSpace bucketSpace) { _bucketSpace = bucketSpace; } diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp index deb38bb927d..f207c4436b1 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.cpp +++ b/storage/src/vespa/storage/visiting/visitormanager.cpp @@ -7,7 +7,6 @@ #include "countvisitor.h" #include "testvisitor.h" #include "recoveryvisitor.h" -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storage/common/statusmessages.h> #include <vespa/config/common/exceptions.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> @@ -46,10 +45,6 @@ VisitorManager::VisitorManager(const config::ConfigUri & configUri, _enforceQueueUse(false), _visitorFactories(externalFactories) { - _component.getMemoryManager().registerAllocationType( - framework::MemoryAllocationType( - "VISITOR_BUFFER", - framework::MemoryAllocationType::EXTERNAL_LOAD)); _configFetcher.subscribe<vespa::config::content::core::StorVisitorConfig>(configUri.getConfigId(), this); _configFetcher.start(); _component.registerMetric(*_metrics); diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index d3fed86b741..4743ac66054 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -2,7 +2,6 @@ #include "visitorthread.h" #include "messages.h" -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/select/bodyfielddetector.h> #include <vespa/document/select/orderingselector.h> @@ -100,9 +99,7 @@ VisitorThread::VisitorThread(uint32_t threadIndex, _timeBetweenTicks(1000), _component(componentRegister, getThreadName(threadIndex)), _messageSessionFactory(messageSessionFac), - _visitorFactories(visitorFactories), - _memoryBufferAlloc( - _component.getMemoryManager().getAllocationType("VISITOR_BUFFER")) + _visitorFactories(visitorFactories) { framework::MilliSecTime maxProcessingTime(30 * 1000); framework::MilliSecTime waitTime(1000); @@ -473,8 +470,6 @@ VisitorThread::onCreateVisitor( cmd->getInstanceId().c_str(), errors.str().c_str()); break; } - visitor->setAllocationType(_memoryBufferAlloc); - visitor->setMemoryManager(_component.getMemoryManager()); // Set visitor parameters if (cmd->getMaximumPendingReplyCount() != 0) { visitor->setMaxPending(cmd->getMaximumPendingReplyCount()); diff --git a/storage/src/vespa/storage/visiting/visitorthread.h b/storage/src/vespa/storage/visiting/visitorthread.h index 1f1459ccb2b..97ca70f3761 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.h +++ b/storage/src/vespa/storage/visiting/visitorthread.h @@ -90,7 +90,6 @@ class VisitorThread : public framework::Runnable, framework::Thread::UP _thread; VisitorMessageSessionFactory& _messageSessionFactory; VisitorFactory::Map& _visitorFactories; - const framework::MemoryAllocationType& _memoryBufferAlloc; public: VisitorThread(uint32_t threadIndex, diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h index 663a0ccdf4e..32f6c769e87 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h @@ -12,7 +12,6 @@ #pragma once #include "messagehandler.h" -#include <vespa/storageframework/generic/memory/memorytoken.h> #include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/messagebus/routing/route.h> #include <vespa/messagebus/trace.h> @@ -335,7 +334,6 @@ private: StorageMessage(const StorageMessage&); mutable std::unique_ptr<TransportContext> _transportContext; - std::unique_ptr<framework::MemoryToken> _memoryToken; protected: static Id generateMsgId(); @@ -375,10 +373,6 @@ public: void setAddress(const StorageMessageAddress& address) { _address.reset(new StorageMessageAddress(address)); } - void setMemoryToken(std::unique_ptr<framework::MemoryToken> token) { - _memoryToken = std::move(token); - } - /** Returns the approximate memory footprint of a storage message. By default, returns 50 bytes. This only needs to be overriden if the diff --git a/storageframework/CMakeLists.txt b/storageframework/CMakeLists.txt index 5a4b2e02372..a3414a8b05c 100644 --- a/storageframework/CMakeLists.txt +++ b/storageframework/CMakeLists.txt @@ -11,12 +11,10 @@ vespa_define_module( src/vespa/storageframework/defaultimplementation src/vespa/storageframework/defaultimplementation/clock src/vespa/storageframework/defaultimplementation/component - src/vespa/storageframework/defaultimplementation/memory src/vespa/storageframework/defaultimplementation/thread src/vespa/storageframework/generic src/vespa/storageframework/generic/clock src/vespa/storageframework/generic/component - src/vespa/storageframework/generic/memory src/vespa/storageframework/generic/metric src/vespa/storageframework/generic/status src/vespa/storageframework/generic/thread @@ -28,7 +26,6 @@ vespa_define_module( TESTS src/tests src/tests/clock - src/tests/memory src/tests/status src/tests/thread ) diff --git a/storageframework/src/tests/CMakeLists.txt b/storageframework/src/tests/CMakeLists.txt index eb6d45330e8..0d604d64f74 100644 --- a/storageframework/src/tests/CMakeLists.txt +++ b/storageframework/src/tests/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_executable(storageframework_testrunner_app TEST DEPENDS storageframework_testclock storageframework_teststatus - storageframework_testmemory storageframework_testthread ) diff --git a/storageframework/src/tests/memory/.gitignore b/storageframework/src/tests/memory/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/storageframework/src/tests/memory/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/storageframework/src/tests/memory/CMakeLists.txt b/storageframework/src/tests/memory/CMakeLists.txt deleted file mode 100644 index 591a78f1046..00000000000 --- a/storageframework/src/tests/memory/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageframework_testmemory - SOURCES - memorymanagertest.cpp - memorystatetest.cpp - DEPENDS - storageframework -) diff --git a/storageframework/src/tests/memory/memorymanagertest.cpp b/storageframework/src/tests/memory/memorymanagertest.cpp deleted file mode 100644 index 5424b8dea82..00000000000 --- a/storageframework/src/tests/memory/memorymanagertest.cpp +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/storageframework/defaultimplementation/clock/realclock.h> -#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> -#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h> -#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/vespalib/util/document_runnable.h> -#include <vespa/vespalib/util/random.h> - -namespace storage { -namespace framework { -namespace defaultimplementation { - -struct MemoryManagerTest : public CppUnit::TestFixture -{ - void testBasics(); - void testCacheAllocation(); - void testStress(); - - CPPUNIT_TEST_SUITE(MemoryManagerTest); - CPPUNIT_TEST(testBasics); - CPPUNIT_TEST(testCacheAllocation); - CPPUNIT_TEST(testStress); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemoryManagerTest); - -void -MemoryManagerTest::testBasics() -{ - uint64_t maxMemory = 1000; - RealClock clock; - SimpleMemoryLogic* logic = new SimpleMemoryLogic(clock, maxMemory); - AllocationLogic::UP allLogic(std::move(logic)); - MemoryManager manager(std::move(allLogic)); - - const MemoryAllocationType& putAlloc(manager.registerAllocationType( - MemoryAllocationType("put", MemoryAllocationType::EXTERNAL_LOAD))); - const MemoryAllocationType& getAlloc(manager.registerAllocationType( - MemoryAllocationType("get", MemoryAllocationType::EXTERNAL_LOAD))); - const MemoryAllocationType& bufAlloc(manager.registerAllocationType( - MemoryAllocationType("buffer"))); - const MemoryAllocationType& cacheAlloc(manager.registerAllocationType( - MemoryAllocationType("cache", MemoryAllocationType::CACHE))); - const MemoryState& state(logic->getState()); - const MemoryState::SnapShot& current(state.getCurrentSnapshot()); - // Basics - { - // * Getting a token, and release it back with correct behavior - framework::MemoryToken::UP put = manager.allocate(putAlloc, - 0, 100, 80); - CPPUNIT_ASSERT(put.get() != 0); - CPPUNIT_ASSERT_EQUAL(uint64_t(100), put->getSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(100), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(900), state.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize()); - - // * Do the same while not being empty. Different type. - framework::MemoryToken::UP get = manager.allocate(getAlloc, - 30, 200, 50); - CPPUNIT_ASSERT(get.get() != 0); - CPPUNIT_ASSERT_EQUAL(uint64_t(200), get->getSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(300), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(700), state.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize()); - - // * Do the same while not being empty. Same type. - framework::MemoryToken::UP get2 = manager.allocate( - getAlloc, - 70, - 150, - 60); - - CPPUNIT_ASSERT(get2.get() != 0); - CPPUNIT_ASSERT_EQUAL(uint64_t(150), get2->getSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(450), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(550), state.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize()); - } - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount()); - - // Non-external load - // * Getting minimum when going beyond 80% full - { - framework::MemoryToken::UP filler = manager.allocate(putAlloc, - 795, 795, 90); - framework::MemoryToken::UP resize = manager.allocate( - bufAlloc, 10, 90, 80); - CPPUNIT_ASSERT(resize.get() != 0); - CPPUNIT_ASSERT_EQUAL(uint64_t(10), resize->getSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(805), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(195), state.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize()); - } - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount()); - - // Non-external load - // * Getting up to threshold if hitting it - { - framework::MemoryToken::UP filler = manager.allocate(putAlloc, - 750, 750, 90); - framework::MemoryToken::UP resize = manager.allocate( - bufAlloc, 10, 90, 80); - CPPUNIT_ASSERT(resize.get() != 0); - CPPUNIT_ASSERT_EQUAL(uint64_t(50), resize->getSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(800), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(200), state.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize()); - } - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount()); - - // External load - { - // * Stopped when going beyond 80% full - framework::MemoryToken::UP filler = manager.allocate(putAlloc, - 795, 795, 90); - framework::MemoryToken::UP put = manager.allocate(putAlloc, - 10, 100, 80); - CPPUNIT_ASSERT(put.get() == 0); - CPPUNIT_ASSERT_EQUAL(uint64_t(795), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(205), state.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize()); - } - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount()); - - // External load - { - // * Getting up to threshold if hitting it - framework::MemoryToken::UP filler = manager.allocate(putAlloc, - 750, 750, 90); - framework::MemoryToken::UP put = manager.allocate(putAlloc, - 10, 100, 80); - CPPUNIT_ASSERT(put.get() != 0); - CPPUNIT_ASSERT_EQUAL(uint64_t(50), put->getSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(800), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(200), state.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize()); - } - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount()); - - // Test caching.. - { - // Cache paradigm: - // Allocate a token taking up no space at all. - // Give it to your ReduceMemoryUsageInterface implementation. - // Run resize on your token in that implementation to get memory and - // return memory. That way locking should be easy when needed. - struct ReduceI : public framework::ReduceMemoryUsageInterface { - framework::MemoryToken::UP _token; - - uint64_t reduceMemoryConsumption(const MemoryToken& token, uint64_t reduceBy) override { - assert(&token == _token.get()); - (void) &token; - assert(_token->getSize() >= reduceBy); - return reduceBy; - } - }; - ReduceI reducer; - framework::MemoryToken::UP cache = manager.allocate(cacheAlloc, - 0, 0, 0, &reducer); - CPPUNIT_ASSERT(cache.get() != 0); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), cache->getSize()); - reducer._token = std::move(cache); - for (uint32_t i=1; i<=50; ++i) { - bool success = reducer._token->resize(i * 10, i * 10); - CPPUNIT_ASSERT_EQUAL(true, success); - } - CPPUNIT_ASSERT_EQUAL(uint64_t(500), reducer._token->getSize()); - - // * Ordered to free space - framework::MemoryToken::UP put = manager.allocate(putAlloc, - 600, 600, 80); - CPPUNIT_ASSERT_EQUAL_MSG(manager.toString(), - uint64_t(400), reducer._token->getSize()); - CPPUNIT_ASSERT_EQUAL_MSG(manager.toString(), - uint64_t(600), put->getSize()); - } - CPPUNIT_ASSERT_EQUAL_MSG(state.toString(true), - uint64_t(0), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL_MSG(state.toString(true), - uint64_t(0), current.getUserCount()); - - // Test merge and tracking of allocation counts with merge, by doing - // operations with tokens and see that user count and used size - // correctly go back to zero. - { - framework::MemoryToken::UP tok1( - manager.allocate(putAlloc, 5, 5, 40)); - framework::MemoryToken::UP tok2( - manager.allocate(putAlloc, 10, 10, 40)); - framework::MemoryToken::UP tok3( - manager.allocate(putAlloc, 20, 20, 40)); - framework::MemoryToken::UP tok4( - manager.allocate(putAlloc, 40, 40, 40)); - framework::MemoryToken::UP tok5( - manager.allocate(putAlloc, 80, 80, 40)); - framework::MemoryToken::UP tok6( - manager.allocate(putAlloc, 1, 1, 40)); - framework::MemoryToken::UP tok7( - manager.allocate(putAlloc, 3, 3, 40)); - } -} - -void -MemoryManagerTest::testCacheAllocation() -{ - uint64_t maxMemory = 3000; - - RealClock clock; - SimpleMemoryLogic::UP logic(new PriorityMemoryLogic(clock, maxMemory)); - logic->setCacheThreshold(1.0); - - AllocationLogic::UP allLogic(std::move(logic)); - MemoryManager manager(std::move(allLogic)); - - const MemoryAllocationType& putAlloc(manager.registerAllocationType( - MemoryAllocationType("put", MemoryAllocationType::EXTERNAL_LOAD))); - const MemoryAllocationType& cacheAlloc(manager.registerAllocationType( - MemoryAllocationType("cache", MemoryAllocationType::CACHE))); - - framework::MemoryToken::UP token = - manager.allocate(putAlloc, - 50, - 50, - 127); - - CPPUNIT_ASSERT_EQUAL(50, (int)token->getSize()); - - framework::MemoryToken::UP token2 = - manager.allocate(cacheAlloc, - 1000, - 2000, - 127); - - CPPUNIT_ASSERT_EQUAL(2000, (int)token2->getSize()); - - token2->resize(2000, 3000); - - CPPUNIT_ASSERT_EQUAL(2950, (int)token2->getSize()); -} - -namespace { -struct MemoryManagerLoadGiver : public document::Runnable, - public ReduceMemoryUsageInterface -{ - MemoryManager& _manager; - const framework::MemoryAllocationType& _type; - uint8_t _priority; - uint32_t _minMem; - uint32_t _maxMem; - uint32_t _failed; - uint32_t _ok; - uint32_t _reduced; - using MemoryTokenUP = std::unique_ptr<MemoryToken>; - std::vector<MemoryTokenUP> _tokens; - vespalib::Lock _cacheLock; - - MemoryManagerLoadGiver( - MemoryManager& manager, - const framework::MemoryAllocationType& type, - uint8_t priority, - uint32_t minMem, - uint32_t maxMem, - uint32_t tokensToKeep) - : _manager(manager), - _type(type), - _priority(priority), - _minMem(minMem), - _maxMem(maxMem), - _failed(0), - _ok(0), - _reduced(0), - _tokens(tokensToKeep) - { - } - - uint64_t reduceMemoryConsumption(const MemoryToken&, uint64_t reduceBy) override { - ++_reduced; - return reduceBy; - } - - void run() override { - ReduceMemoryUsageInterface* reducer = 0; - if (_type.isCache()) reducer = this; - vespalib::RandomGen randomizer; - while (running()) { - vespalib::Lock lock(_cacheLock); - framework::MemoryToken::UP token = _manager.allocate( - _type, _minMem, _maxMem, _priority, reducer); - if (token.get() == 0) { - ++_failed; - } else { - ++_ok; - } - uint32_t index = randomizer.nextUint32(0, _tokens.size() - 1); - _tokens[index] = MemoryTokenUP(token.release()); - } - } -}; -} - -void -MemoryManagerTest::testStress() -{ - uint64_t stressTimeMS = 1 * 1000; - uint64_t maxMemory = 1 * 1024 * 1024; - RealClock clock; - AllocationLogic::UP logic(new PriorityMemoryLogic(clock, maxMemory)); - MemoryManager manager(std::move(logic)); - - FastOS_ThreadPool pool(128 * 1024); - std::vector<MemoryManagerLoadGiver*> loadGivers; - for (uint32_t type = 0; type < 5; ++type) { - const MemoryAllocationType* allocType = 0; - uint32_t min = 1000, max = 5000; - if (type == 0) { - allocType = &manager.registerAllocationType(MemoryAllocationType( - "default")); - } else if (type == 1) { - allocType = &manager.registerAllocationType(MemoryAllocationType( - "external", MemoryAllocationType::EXTERNAL_LOAD)); - } else if (type == 2) { - allocType = &manager.registerAllocationType(MemoryAllocationType( - "forced", MemoryAllocationType::FORCE_ALLOCATE)); - } else if (type == 3) { - allocType = &manager.registerAllocationType(MemoryAllocationType( - "forcedExternal", MemoryAllocationType::FORCE_ALLOCATE - | MemoryAllocationType::EXTERNAL_LOAD)); - } else if (type == 4) { - allocType = &manager.registerAllocationType(MemoryAllocationType( - "cache", MemoryAllocationType::CACHE)); - max = 30000; - } - for (int priority = 0; priority < 256; priority += 8) { - loadGivers.push_back(new MemoryManagerLoadGiver( - manager, *allocType, priority, min, max, 10)); - loadGivers.back()->start(pool); - } - FastOS_Thread::Sleep(stressTimeMS); - } - FastOS_Thread::Sleep(5 * stressTimeMS); - uint64_t okTotal = 0, failedTotal = 0, reducedTotal = 0; - for (uint32_t i = 0; i < loadGivers.size(); i++) { - /* - fprintf(stderr, "%d %s-%u: Failed %d, ok %d, reduced %d\n", - i, loadGivers[i]->_type.getName().c_str(), - uint32_t(loadGivers[i]->_priority), - loadGivers[i]->_failed, loadGivers[i]->_ok, - loadGivers[i]->_reduced); // */ - okTotal += loadGivers[i]->_ok; - failedTotal += loadGivers[i]->_failed; - reducedTotal += loadGivers[i]->_reduced; - } - for (uint32_t i = 0; i < loadGivers.size(); i++) loadGivers[i]->stop(); - for (uint32_t i = 0; i < loadGivers.size(); i++) loadGivers[i]->join(); - pool.Close(); - - /* - bool verbose = false; - std::cerr << "\n\nMemory allocations at end of load:\n"; - manager.print(std::cerr, verbose, ""); // */ - - for (uint32_t i = 0; i < loadGivers.size(); i++) { - loadGivers[i]->_tokens.clear(); - } - for (uint32_t i = 0; i < loadGivers.size(); i++) { - delete loadGivers[i]; - } - loadGivers.clear(); - - //std::cerr << "\n\nMemory allocations at end of testl:\n"; - //manager.print(std::cerr, verbose, ""); - - std::cerr << "\n Managed " << std::fixed - << (okTotal / (stressTimeMS / 1000)) - << " ok, " << (failedTotal / (stressTimeMS / 1000)) - << " failed and " << (reducedTotal / (stressTimeMS / 1000)) - << " reduced allocations/s.\n "; - - MemoryState state(clock, 1); - manager.getState(state); - const MemoryState::SnapShot& current(state.getCurrentSnapshot()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSizeIgnoringCache()); -} - -} // defaultimplementation -} // framework -} // storage diff --git a/storageframework/src/tests/memory/memorystatetest.cpp b/storageframework/src/tests/memory/memorystatetest.cpp deleted file mode 100644 index cd565718632..00000000000 --- a/storageframework/src/tests/memory/memorystatetest.cpp +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/storageframework/defaultimplementation/clock/realclock.h> -#include <vespa/storageframework/defaultimplementation/memory/memorystate.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/vespalib/util/exceptions.h> - -namespace storage { -namespace framework { -namespace defaultimplementation { - -struct MemoryStateTest : public CppUnit::TestFixture -{ - - void testBasics(); - - CPPUNIT_TEST_SUITE(MemoryStateTest); - CPPUNIT_TEST(testBasics); // Fails sometimes, test needs rewrite. - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemoryStateTest); - -class SimpleMemoryManager : public framework::MemoryManagerInterface -{ -private: - std::map<std::string, framework::MemoryAllocationType> _types; - -public: - void setMaximumMemoryUsage(uint64_t max) override { (void) max; } - - const framework::MemoryAllocationType& - registerAllocationType(const framework::MemoryAllocationType& type) override { - _types[type.getName()] = type; - return _types[type.getName()]; - } - - const framework::MemoryAllocationType& - getAllocationType(const std::string& name) const override { - std::map<std::string, framework::MemoryAllocationType>::const_iterator iter = - _types.find(name); - - if (iter == _types.end()) { - throw vespalib::IllegalArgumentException("Allocation type not found: " + name); - } - - return iter->second; - } - - std::vector<const MemoryAllocationType*> getAllocationTypes() const override { - std::vector<const MemoryAllocationType*> types; - for(std::map<std::string, framework::MemoryAllocationType> - ::const_iterator it = _types.begin(); it != _types.end(); ++it) - { - types.push_back(&it->second); - } - return types; - } - - framework::MemoryToken::UP allocate(const framework::MemoryAllocationType&, - uint64_t, - uint64_t, - uint8_t, - framework::ReduceMemoryUsageInterface*) override - { - return framework::MemoryToken::UP(); - } - - uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override { - (void) priority; - return 0; - } -}; - -void -MemoryStateTest::testBasics() -{ - SimpleMemoryManager manager; - - const MemoryAllocationType& putAlloc(manager.registerAllocationType( - MemoryAllocationType("MESSAGE_PUT", MemoryAllocationType::EXTERNAL_LOAD))); - const MemoryAllocationType& getAlloc(manager.registerAllocationType( - MemoryAllocationType("MESSAGE_GET", MemoryAllocationType::EXTERNAL_LOAD))); - const MemoryAllocationType& blockAlloc(manager.registerAllocationType( - MemoryAllocationType("MESSAGE_DOCBLOCK"))); - const MemoryAllocationType& databaseAlloc(manager.registerAllocationType( - MemoryAllocationType("DATABASE"))); - const MemoryAllocationType& cacheAlloc(manager.registerAllocationType( - MemoryAllocationType("SLOTFILE_CACHE", MemoryAllocationType::CACHE))); - - uint32_t maxMemory = 1024; - - RealClock clock; - MemoryState state1(clock, maxMemory); - MemoryState state2(clock, maxMemory); - - state1.setMinJumpToUpdateMax(50); - - state1.addToEntry(putAlloc, 100, 10, - MemoryState::GOT_MAX, false); - state1.addToEntry(putAlloc, 100, 60, - MemoryState::GOT_MAX, false); - state1.addToEntry(blockAlloc, - 200, 20, - MemoryState::GOT_MIN, false); - state1.addToEntry(getAlloc, 0, 15, - MemoryState::DENIED, false, 0); - state1.addToEntry(databaseAlloc, 150, 0, - MemoryState::DENIED, true, 1); - state1.addToEntry(cacheAlloc, 45, 0, - MemoryState::GOT_MAX, true, 1); - - state2.addToEntry(putAlloc, 50, 10, - MemoryState::GOT_MIN, false); - state2.addToEntry(putAlloc, 20, 40, - MemoryState::GOT_MIN, false); - - state1.removeFromEntry(databaseAlloc, 25, 0, 0); - state1.removeFromEntry(putAlloc, 100, 60); - - MemoryState::SnapShot state3; - state3 = state1.getMaxSnapshot(); - state3 += state2.getMaxSnapshot(); - - std::string expected; - expected = - "\n" - "MemoryState(Max memory: 1024) {\n" - " Current: SnapShot(Used 470, w/o cache 425) {\n" - " Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n" - " DATABASE(0): Used(125 B / 1) Stats(1, 0, 0, 1, 1)\n" - " MESSAGE_DOCBLOCK(20): Used(200 B / 1) Stats(1, 0, 1, 0, 0)\n" - " MESSAGE_GET(15): Used(0 B / 0) Stats(1, 0, 0, 1, 0)\n" - " MESSAGE_PUT(10): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n" - " MESSAGE_PUT(60): Used(0 B / 0) Stats(1, 1, 0, 0, 0)\n" - " SLOTFILE_CACHE(0): Used(45 B / 1) Stats(1, 1, 0, 0, 1)\n" - " }\n" - " Max: SnapShot(Used 550, w/o cache 550) {\n" - " Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n" - " DATABASE(0): Used(150 B / 1) Stats(1, 0, 0, 1, 1)\n" - " MESSAGE_DOCBLOCK(20): Used(200 B / 1) Stats(1, 0, 1, 0, 0)\n" - " MESSAGE_GET(15): Used(0 B / 0) Stats(1, 0, 0, 1, 0)\n" - " MESSAGE_PUT(10): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n" - " MESSAGE_PUT(60): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n" - " }\n" - "}"; - - CPPUNIT_ASSERT_EQUAL(expected, "\n" + state1.toString(true)); - expected = "\n" -"MemoryState(Max memory: 1024) {\n" -" Current: SnapShot(Used 70, w/o cache 70) {\n" -" Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n" -" MESSAGE_PUT(10): Used(50 B / 1) Stats(1, 0, 1, 0, 0)\n" -" MESSAGE_PUT(40): Used(20 B / 1) Stats(1, 0, 1, 0, 0)\n" -" }\n" -"}"; - CPPUNIT_ASSERT_EQUAL(expected, "\n" + state2.toString(true)); - expected = "\n" -"SnapShot(Used 550, w/o cache 550) {\n" -" Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n" -" DATABASE(0): Used(150 B / 1) Stats(1, 0, 0, 1, 1)\n" -" MESSAGE_DOCBLOCK(20): Used(200 B / 1) Stats(1, 0, 1, 0, 0)\n" -" MESSAGE_GET(15): Used(0 B / 0) Stats(1, 0, 0, 1, 0)\n" -" MESSAGE_PUT(10): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n" -" MESSAGE_PUT(60): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n" -"}"; - CPPUNIT_ASSERT_EQUAL(expected, "\n" + state3.toString(true)); -} - -} // defaultimplementation -} // framework -} // storage diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt b/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt index 25e7a9812cf..ea6a7e15543 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt +++ b/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(storageframework_defaultimplementation SOURCES $<TARGET_OBJECTS:storageframework_clockimpl> $<TARGET_OBJECTS:storageframework_componentimpl> - $<TARGET_OBJECTS:storageframework_memoryimpl> $<TARGET_OBJECTS:storageframework_threadimpl> INSTALL lib64 DEPENDS diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp index a03f79b008d..84b091d25e3 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp @@ -63,16 +63,6 @@ ComponentRegisterImpl::setMetricManager(metrics::MetricManager& mm) } void -ComponentRegisterImpl::setMemoryManager(MemoryManagerInterface& mm) -{ - vespalib::LockGuard lock(_componentLock); - _memoryManager = &mm; - for (uint32_t i=0; i<_components.size(); ++i) { - _components[i]->setMemoryManager(mm); - } -} - -void ComponentRegisterImpl::setClock(Clock& c) { vespalib::LockGuard lock(_componentLock); diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h index 7319a1e7d57..3e3dd08e1df 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h @@ -71,7 +71,6 @@ public: void requestShutdown(vespalib::stringref reason) override; void setMetricManager(metrics::MetricManager&); - void setMemoryManager(MemoryManagerInterface&); void setClock(Clock&); void setThreadPool(ThreadPool&); void setUpgradeFlag(UpgradeFlags flag); diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp index 0f4d0ebe070..5efd638ec26 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp @@ -7,13 +7,9 @@ namespace storage::framework::defaultimplementation { TestComponentRegister::TestComponentRegister(ComponentRegisterImpl::UP compReg) : _compReg(std::move(compReg)), _clock(), - _threadPool(_clock), - _memoryManager() + _threadPool(_clock) { assert(_compReg.get() != 0); - // Set a memory manager, so users can register memory types and - // ask for memory. - _compReg->setMemoryManager(_memoryManager); // Set a fake clock, giving test control of clock _compReg->setClock(_clock); // Set a thread pool so components can make threads in tests. diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h index 5226765dc40..4983d5dcdfc 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h @@ -15,7 +15,6 @@ #include "componentregisterimpl.h" #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> -#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> namespace storage::framework::defaultimplementation { @@ -24,7 +23,6 @@ class TestComponentRegister { ComponentRegisterImpl::UP _compReg; FakeClock _clock; ThreadPoolImpl _threadPool; - NoMemoryManager _memoryManager; public: TestComponentRegister(ComponentRegisterImpl::UP compReg); @@ -34,7 +32,6 @@ public: FakeClock& getClock() { return _clock; } ThreadPoolImpl& getThreadPoolImpl() { return _threadPool; } FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); } - NoMemoryManager& getMemoryManager() { return _memoryManager; } }; } diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore b/storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt b/storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt deleted file mode 100644 index e0aa10a6716..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageframework_memoryimpl OBJECT - SOURCES - memorymanager.cpp - simplememorylogic.cpp - memorystate.cpp - prioritymemorylogic.cpp - nomemorymanager.cpp - DEPENDS -) diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h deleted file mode 100644 index 5af2c9ff929..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <storage/memorymanager/memorymanager.h> -#include <storage/memorymanager/memorystate.h> - -namespace storage { - -class EmptyMemoryLogic : public AllocationLogic -{ -private: - MemoryState _state; - -public: - EmptyMemoryLogic() : _state(100) {} - - virtual void getState(MemoryState& state, bool resetMax) { - state = _state; - if (resetMax) _state.resetMax(); - } - - virtual MemoryToken::UP allocate( - const AllocationType& type, - uint64_t, uint64_t max, - storage::api::StorageMessage::Priority p, - ReduceMemoryUsageInterface* = 0) { - return MemoryToken::UP( - new MemoryToken(*this, type, max, p)); - } - - virtual bool resize(MemoryToken& token, uint64_t min, uint64_t max) { - setTokenSize(token, max); - return true; - } - - virtual void freeToken(MemoryToken&) {} - - virtual void print(std::ostream& out, bool verbose, - const std::string& indent) const - { - (void) verbose; (void) indent; - out << "EmptyMemoryLogic()"; - } -}; - -} // storage - diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp deleted file mode 100644 index 5af222d4b36..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memorymanager.h" -#include "memorystate.h" -#include <vespa/vespalib/util/exceptions.h> - -namespace storage { -namespace framework { -namespace defaultimplementation { - -MemoryTokenImpl::MemoryTokenImpl(AllocationLogic& logic, - const MemoryAllocationType& type, - uint64_t allocated, - uint8_t p, - ReduceMemoryUsageInterface* reducer) - : _logic(logic), - _reducer(reducer), - _currentlyAllocated(allocated), - _allocCount(1), - _type(type), - _priority(p) -{ -} - -MemoryTokenImpl::~MemoryTokenImpl() -{ - _logic.freeToken(*this); -} - -bool -MemoryTokenImpl::resize(uint64_t min, uint64_t max) -{ - return _logic.resize(*this, min, max); -} - -void -MemoryTokenImpl::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "MemoryToken(" << _type.getName() << ": Allocated(" << _allocCount << " - " - << _currentlyAllocated << ")"; -} - -AllocationLogic::~AllocationLogic() -{ -} - -MemoryToken::UP -AllocationLogic::allocate(const MemoryAllocationType& type, - uint64_t min, - uint64_t max, - uint8_t priority, - ReduceMemoryUsageInterface* reducer) -{ - MemoryToken::UP token(allocate(type, priority, reducer)); - assert(token.get()); - if (!resize(*token, min, max, 1)) token.reset(); - return token; -} - -bool -AllocationLogic::resize(MemoryTokenImpl& token, uint64_t min, uint64_t max) -{ - return resize(token, min, max, 0); -} - -MemoryManager::MemoryManager(AllocationLogic::UP logic) - : _logic(std::move(logic)) -{ - if (_logic.get() == 0) { - throw vespalib::IllegalArgumentException( - "Needs a real logic class to run. (Got null pointer)", - VESPA_STRLOC); - } -} - -MemoryManager::~MemoryManager() -{ -} - -void -MemoryManager::setMaximumMemoryUsage(uint64_t max) -{ - _logic->setMaximumMemoryUsage(max); -} - -void -MemoryManager::getState(MemoryState& state, bool resetMax) -{ - return _logic->getState(state, resetMax); -} - -const MemoryAllocationType& -MemoryManager::registerAllocationType(const MemoryAllocationType& type) -{ - vespalib::LockGuard lock(_typeLock); - _types[type.getName()] = MemoryAllocationType::UP( - new MemoryAllocationType(type)); - return *_types[type.getName()]; -} - -const MemoryAllocationType& -MemoryManager::getAllocationType(const std::string& name) const -{ - vespalib::LockGuard lock(_typeLock); - std::map<std::string, MemoryAllocationType::UP>::const_iterator it( - _types.find(name)); - if (it == _types.end()) { - throw vespalib::IllegalArgumentException( - "Allocation type not found: " + name, VESPA_STRLOC); - } - return *it->second; -} - -std::vector<const MemoryAllocationType*> -MemoryManager::getAllocationTypes() const -{ - vespalib::LockGuard lock(_typeLock); - std::vector<const MemoryAllocationType*> types; - for(std::map<std::string, MemoryAllocationType::UP>::const_iterator it - = _types.begin(); it != _types.end(); ++it) - { - types.push_back(it->second.get()); - } - return types; -} - -MemoryToken::UP -MemoryManager::allocate(const MemoryAllocationType& type, - uint64_t min, - uint64_t max, - uint8_t p, - ReduceMemoryUsageInterface* i) -{ - return _logic->allocate(type, min, max, p, i); -} - -uint64_t -MemoryManager::getMemorySizeFreeForPriority(uint8_t priority) const -{ - return _logic->getMemorySizeFreeForPriority(priority); -} - -void -MemoryManager::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "Memory Manager {" << "\n" << indent << " "; - _logic->print(out, verbose, indent + " "); - out << "\n" << indent << "}"; -} - -} // defaultimplementation -} // framework -} // storage diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h deleted file mode 100644 index 86a2920095e..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::MemoryManager - * - * \brief Utility for tracking memory usage on distributor and storage nodes. - * - * The memory manager is responsible for limiting the memory allocated by - * various users within VDS storage and distributor nodes, such that these - * nodes don't use more memory than they can use, to avoid swapping. - * - * It will produce a status page to view big memory users, and give some - * historic data. It will track memory users and give them less and less memory - * as closer we are to utilizing all memory we are able to use. When getting - * close to full it will deny memory allocations to incoming commands that wants - * to use additional memory in able to complete the operation. - * - * The main class here defines the interface the client has to worry about. It - * should thus not point to the implementation in any way. - * - */ - -#pragma once - -#include <map> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> -#include <vespa/vespalib/util/printable.h> -#include <vespa/vespalib/util/sync.h> - -namespace storage::framework::defaultimplementation { - -class MemoryManager; -class AllocationLogic; -class MemoryState; - -class MemoryTokenImpl : public vespalib::Printable, - public MemoryToken -{ - friend class AllocationLogic; - - AllocationLogic& _logic; - ReduceMemoryUsageInterface* _reducer; - uint64_t _currentlyAllocated; - uint32_t _allocCount; - const MemoryAllocationType& _type; - uint8_t _priority; - -public: - typedef std::unique_ptr<MemoryTokenImpl> UP; - - MemoryTokenImpl(const MemoryTokenImpl &) = delete; - MemoryTokenImpl & operator = (const MemoryTokenImpl &) = delete; - MemoryTokenImpl(AllocationLogic& logic, - const MemoryAllocationType& type, - uint64_t allocated, - uint8_t priority, - ReduceMemoryUsageInterface* = 0); - - ~MemoryTokenImpl(); - - uint64_t getSize() const override { return _currentlyAllocated; } - uint64_t getAllocationCount() const { return _allocCount; } - const MemoryAllocationType& getType() const { return _type; } - ReduceMemoryUsageInterface* getReducer() const { return _reducer; } - - uint8_t getPriority() const { return _priority; } - - bool resize(uint64_t min, uint64_t max) override; - - void print(std::ostream& out, bool verbose, - const std::string& indent) const override; -}; - -class AllocationLogic : public vespalib::Printable -{ -protected: - /** - * MemoryTokens are friends with this class, such that logic classes - * can use this function to alter token size, without function for that - * being public. - */ - void setTokenSize(MemoryTokenImpl& token, uint64_t size) - { token._currentlyAllocated = size; } - - virtual MemoryToken::UP allocate(const MemoryAllocationType&, - uint8_t priority, - ReduceMemoryUsageInterface*) = 0; - virtual bool resize(MemoryToken& token, uint64_t min, uint64_t max, - uint32_t allocationCounts) = 0; -public: - typedef std::unique_ptr<AllocationLogic> UP; - virtual ~AllocationLogic() = 0; - - virtual void setMaximumMemoryUsage(uint64_t max) = 0; - - virtual void getState(MemoryState&, bool resetMax = false) = 0; - - /** - * Decide how much to allocate for this request. Should be between min - * and max, unless it's of a type that can be denied (such as external - * requests), in which case we can also deny allocation by returning a null - * token. - */ - MemoryToken::UP allocate(const MemoryAllocationType&, - uint64_t min, - uint64_t max, - uint8_t priority, - ReduceMemoryUsageInterface* = 0); - /** - * Resize the size in a token. If more memory is requested, then it might - * fail. The sizes given in min and max is given as total min and max, - * including any memory you may already have. If successful, the logic will - * have added this size to the token passed in. - */ - bool resize(MemoryTokenImpl& token, uint64_t min, uint64_t max); - - // Called by token destructor to free up tracked resources - virtual void freeToken(MemoryTokenImpl& token) = 0; - - virtual uint64_t getMemorySizeFreeForPriority(uint8_t priority) const = 0; - - // vespalib::Printable implementation - virtual void print(std::ostream& out, bool verbose, - const std::string& indent) const override = 0; -}; - -class MemoryManager : public vespalib::Printable, - public MemoryManagerInterface -{ - AllocationLogic::UP _logic; - vespalib::Lock _typeLock; - std::map<std::string, MemoryAllocationType::UP> _types; - -public: - typedef std::unique_ptr<MemoryManager> UP; - - MemoryManager(AllocationLogic::UP); - ~MemoryManager(); - - void setMaximumMemoryUsage(uint64_t max) override; - virtual void getState(MemoryState& state, bool resetMax = false); - - const MemoryAllocationType®isterAllocationType(const MemoryAllocationType& type) override; - const MemoryAllocationType&getAllocationType(const std::string& name) const override; - - std::vector<const MemoryAllocationType*> getAllocationTypes() const override; - - MemoryToken::UP allocate(const MemoryAllocationType&, uint64_t min, uint64_t max, - uint8_t p, ReduceMemoryUsageInterface* = 0) override; - - uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - -}; - -} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp deleted file mode 100644 index 2351edd267c..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memorystate.h" -#include <sstream> - -#include <vespa/log/log.h> -LOG_SETUP(".memory.state"); - -namespace storage::framework::defaultimplementation { - -MemoryState::Entry::Entry() - : _currentUsedSize(0), - _totalUserCount(0), - _currentUserCount(0), - _wantedCount(0), - _minimumCount(0), - _deniedCount(0), - _forcedBeyondMaximumCount(0) -{ -} - -void -MemoryState::Entry::operator+=(const Entry& other) -{ - _currentUsedSize += other._currentUsedSize; - _currentUserCount += other._currentUserCount; - _totalUserCount += other._totalUserCount; - _wantedCount += other._wantedCount; - _minimumCount += other._minimumCount; - _deniedCount += other._deniedCount; - _forcedBeyondMaximumCount += other._forcedBeyondMaximumCount; -} - -MemoryState::SnapShot& -MemoryState::SnapShot::operator+=(const MemoryState::SnapShot& other) -{ - for (AllocationMap::const_iterator it = other._allocations.begin(); - it != other._allocations.end(); ++it) - { - PriorityMap& map(_allocations[it->first]); - for (PriorityMap::const_iterator it2 = it->second.begin(); - it2 != it->second.end(); ++it2) - { - Entry& entry(map[it2->first]); - entry += it2->second; - } - } - return *this; -} - -uint64_t -MemoryState::SnapShot::getUserCount() const -{ - uint64_t count = 0; - for (AllocationMap::const_iterator it = _allocations.begin(); - it != _allocations.end(); ++it) - { - for (PriorityMap::const_iterator it2 = it->second.begin(); - it2 != it->second.end(); ++it2) - { - count += it2->second._currentUserCount; - } - } - return count; -} - -MemoryState::MemoryState(Clock& clock, uint64_t maxMemory) - : _clock(&clock), - _maxMemory(maxMemory), - _current(), - _max(), - _minJumpToUpdateMax(10 * 1024 * 1024) -{ -} - -MemoryState::MemoryState(const MemoryState &) = default; -MemoryState & MemoryState::operator = (const MemoryState &) = default; - -MemoryState::~MemoryState() {} - -void -MemoryState::addToEntry(const MemoryAllocationType& type, uint64_t memory, - uint8_t priority, - AllocationResult result, bool forcedAllocation, - uint64_t allocationCounts) -{ - LOG(spam, "Allocating memory %s - %lu bytes at priority %u. " - "Count %lu.", - type.getName().c_str(), memory, priority, allocationCounts); - PriorityMap& map(_current._allocations[&type]); - Entry& e(map[priority]); - e._currentUsedSize += memory; - e._totalUserCount += allocationCounts; - if (allocationCounts == 0) { - // Resizes adding no more users still count as another total - // allocation attempt. - ++e._totalUserCount; - } - e._currentUserCount += allocationCounts; - switch (result) { - case GOT_MAX: ++e._wantedCount; break; - case GOT_MIN: ++e._minimumCount; break; - case DENIED: ++e._deniedCount; break; - } - if (forcedAllocation) ++e._forcedBeyondMaximumCount; - _current._usedMemory += memory; - if (!type.isCache()) { - _current._usedWithoutCache += memory; - } - if (_current._usedWithoutCache - > _max._usedWithoutCache + _minJumpToUpdateMax) - { - LOG(spam, "Updating max to current %lu bytes of memory used", - _current._usedWithoutCache); - _max = _current; - _max._timeTaken = _clock->getTimeInSeconds(); - } -} - -void -MemoryState::removeFromEntry(const MemoryAllocationType& type, uint64_t memory, - uint8_t priority, - uint64_t allocationCounts) -{ - LOG(spam, "Freeing memory %s - %lu bytes at priority %u. " - "Count %lu.", - type.getName().c_str(), memory, priority, allocationCounts); - PriorityMap& map(_current._allocations[&type]); - Entry& e(map[priority]); - e._currentUsedSize -= memory; - e._currentUserCount -= allocationCounts; - _current._usedMemory -= memory; - if (!type.isCache()) { - _current._usedWithoutCache -= memory; - } -} - -void -MemoryState::Entry::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - std::ostringstream ost; - ost << "Used(" << _currentUsedSize << " B / " - << _currentUserCount << ") "; - for (uint32_t i=ost.str().size(); i<20; ++i) { - ost << " "; - } - - out << ost.str() - << "Stats(" << _totalUserCount - << ", " << _wantedCount - << ", " << _minimumCount - << ", " << _deniedCount - << ", " << _forcedBeyondMaximumCount << ")"; -} - -namespace { - void printAllocations(std::ostream& out, - const MemoryState::AllocationMap& map, - const std::string& indent) - { - std::map<std::string, std::string> allocs; - - for (MemoryState::AllocationMap::const_iterator it = map.begin(); - it != map.end(); ++it) - { - for (MemoryState::PriorityMap::const_iterator it2 - = it->second.begin(); it2 != it->second.end(); ++it2) - { - std::ostringstream name; - name << it->first->getName() << "(" - << static_cast<uint16_t>(it2->first) << "): "; - for (uint32_t i=name.str().size(); i<25; ++i) { - name << " "; - } - - std::ostringstream tmp; - it2->second.print(tmp, true, indent + " "); - - allocs[name.str()] = tmp.str(); - } - } - - for (std::map<std::string, std::string>::const_iterator it - = allocs.begin(); it != allocs.end(); ++it) - { - out << "\n" << indent << " " << it->first << it->second; - } - } -} - -void -MemoryState::SnapShot::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SnapShot(Used " << _usedMemory << ", w/o cache " - << _usedWithoutCache; - if (verbose) { - out << ") {"; - if (_usedMemory > 0) { - out << "\n" << indent << " Type(Pri): Used(Size/Allocs) " - << "Stats(Allocs, Wanted, Min, Denied, Forced)"; - } - printAllocations(out, _allocations, indent); - out << "\n" << indent << "}"; - } -} - -void -MemoryState::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - bool maxSet = (_max._usedWithoutCache > _current._usedWithoutCache); - out << "MemoryState(Max memory: " << _maxMemory << ") {" - << "\n" << indent << " Current: "; - _current.print(out, verbose, indent + " "); - if (maxSet) { - out << "\n" << indent << " Max: "; - _max.print(out, verbose, indent + " "); - } - out << "\n" << indent << "}"; -} - -} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h deleted file mode 100644 index 3d670d43844..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::MemoryState - * - * \brief Shows the state of current memory users - * - */ - -#pragma once - -#include "memorymanager.h" -#include <vespa/storageframework/generic/clock/clock.h> -#include <vespa/vespalib/util/sync.h> - -namespace storage::framework::defaultimplementation { - -class MemoryState : public vespalib::Printable { -public: - struct Entry { - // Total number of bytes allocated to this entry right now - uint64_t _currentUsedSize; - // Total number of allocations done on this entry - uint64_t _totalUserCount; - // Total number of allocations for this entry right now - uint32_t _currentUserCount; - // Amount of times this entry has gotten all the memory it wanted - uint32_t _wantedCount; - // Amount of times this entry has gotten less than all the memory - // it wanted - uint32_t _minimumCount; - // Amount of times this entry has been denied getting memory - uint32_t _deniedCount; - // Amount of times this entry has forced memory allocations beyond - // the maximum - uint32_t _forcedBeyondMaximumCount; - - Entry(); - - void print(std::ostream& out, bool verbose, - const std::string& indent) const; - - /** - * Set this instances counts to the counts from the other entry. - */ - void transferCounts(const Entry& other); - - void operator+=(const Entry& other); - }; - - typedef std::map<uint8_t, Entry> PriorityMap; - typedef std::map<const MemoryAllocationType*, PriorityMap> AllocationMap; - - /** - * A snapshot contains data for either current or max seen data. - * When a new maximum is seen, current is copied to max. - */ - class SnapShot : public vespalib::Printable { - friend class MemoryState; - - uint64_t _usedMemory; - uint64_t _usedWithoutCache; - SecondTime _timeTaken; - AllocationMap _allocations; - - public: - SnapShot() : vespalib::Printable() { clear(); } - SnapShot(const SnapShot& o) : vespalib::Printable() { (*this) = o; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - void clear() { - _usedMemory = 0; - _usedWithoutCache = 0; - _timeTaken.setTime(0); - _allocations.clear(); - } - - SnapShot& operator=(const SnapShot& other) { - _usedMemory = other._usedMemory; - _usedWithoutCache = other._usedWithoutCache; - _timeTaken = other._timeTaken; - _allocations = other._allocations; - return *this; - } - - SnapShot& operator+=(const SnapShot& other); - - const AllocationMap& getAllocations() const { return _allocations; } - uint64_t getUsedSize() const { return _usedMemory; } - uint64_t getUsedSizeIgnoringCache() const { return _usedWithoutCache; } - uint64_t getUserCount() const; - }; - -private: - Clock* _clock; - uint64_t _maxMemory; - SnapShot _current; - SnapShot _max; - uint32_t _minJumpToUpdateMax; - -public: - MemoryState(Clock& clock, uint64_t maxMemory); - MemoryState(const MemoryState &); - MemoryState & operator = (const MemoryState &); - MemoryState(MemoryState &&) = default; - MemoryState & operator = (MemoryState &&) = default; - ~MemoryState(); - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - void setMaximumMemoryUsage(uint64_t max) { _maxMemory = max; } - void setMinJumpToUpdateMax(uint32_t bytes) { _minJumpToUpdateMax = bytes; } - - enum AllocationResult { GOT_MAX, GOT_MIN, DENIED }; - void addToEntry(const MemoryAllocationType& type, uint64_t memory, - uint8_t priority, - AllocationResult result, bool forcedAllocation = false, - uint64_t allocationCounts = 1); - - void removeFromEntry(const MemoryAllocationType& type, uint64_t memory, - uint8_t priority, - uint64_t allocationCounts = 1); - void resetMax() { - _max = _current; - _max._timeTaken = _clock->getTimeInSeconds(); - } - - const SnapShot& getCurrentSnapshot() const { return _current; } - const SnapShot& getMaxSnapshot() const { return _max; } - - - uint64_t getTotalSize() const { return _maxMemory; } - uint64_t getFreeSize() const { - return _maxMemory > _current._usedMemory - ? _maxMemory - _current._usedMemory : 0; - } -}; - -} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp deleted file mode 100644 index 16e03240a4f..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "nomemorymanager.h" -#include <vespa/vespalib/util/exceptions.h> - -namespace storage::framework::defaultimplementation { - -NoMemoryManager::~NoMemoryManager() {} - -const MemoryAllocationType& -NoMemoryManager::registerAllocationType(const MemoryAllocationType& type) -{ - vespalib::LockGuard lock(_typeLock); - _types[type.getName()] = MemoryAllocationType::UP( - new MemoryAllocationType(type)); - return *_types[type.getName()]; -} - -const MemoryAllocationType& -NoMemoryManager::getAllocationType(const std::string& name) const -{ - vespalib::LockGuard lock(_typeLock); - std::map<std::string, MemoryAllocationType::UP>::const_iterator it( - _types.find(name)); - if (it == _types.end()) { - throw vespalib::IllegalArgumentException( - "Allocation type not found: " + name, VESPA_STRLOC); - } - return *it->second; -} - -std::vector<const MemoryAllocationType*> -NoMemoryManager::getAllocationTypes() const -{ - vespalib::LockGuard lock(_typeLock); - std::vector<const MemoryAllocationType*> types; - for(std::map<std::string, MemoryAllocationType::UP>::const_iterator it - = _types.begin(); it != _types.end(); ++it) - { - types.push_back(it->second.get()); - } - return types; -} - -} - diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h deleted file mode 100644 index 7f90b1360b8..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::NoMemoryManager - * - * \brief Memory manager that gives out max memory to everyone. - * - * Memory manager to use for testing and for apps not wanting to track memory. - * This manager will merely give out max to everyone who asks and not even keep - * track of anything. - */ - -#pragma once - -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> -#include <vespa/vespalib/util/printable.h> -#include <vespa/vespalib/util/sync.h> -#include <map> - -namespace storage { -namespace framework { -namespace defaultimplementation { - -class SimpleMemoryTokenImpl : public MemoryToken -{ - uint64_t _allocated; - -public: - SimpleMemoryTokenImpl(const SimpleMemoryTokenImpl &) = delete; - SimpleMemoryTokenImpl & operator = (const SimpleMemoryTokenImpl &) = delete; - SimpleMemoryTokenImpl(uint64_t allocated) : _allocated(allocated) {} - - uint64_t getSize() const override { return _allocated; } - bool resize(uint64_t /* min */, uint64_t max) override { _allocated = max; return true; } -}; - -class NoMemoryManager : public MemoryManagerInterface -{ - vespalib::Lock _typeLock; - std::map<std::string, MemoryAllocationType::UP> _types; - -public: - typedef std::unique_ptr<NoMemoryManager> UP; - - ~NoMemoryManager(); - - void setMaximumMemoryUsage(uint64_t) override {} - const MemoryAllocationType & registerAllocationType(const MemoryAllocationType& type) override; - const MemoryAllocationType & getAllocationType(const std::string& name) const override; - - MemoryToken::UP allocate(const MemoryAllocationType&, uint64_t /* min */, uint64_t max, - uint8_t /* priority */, ReduceMemoryUsageInterface* = 0) override - { - return SimpleMemoryTokenImpl::UP(new SimpleMemoryTokenImpl(max)); - } - uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override { - (void) priority; - return std::numeric_limits<uint64_t>().max(); - } - - std::vector<const MemoryAllocationType*> getAllocationTypes() const override; -}; - -} // defaultimplementation -} // framework -} // storage - diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp deleted file mode 100644 index 51bb7ba5019..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "prioritymemorylogic.h" - -#include <vespa/log/log.h> -LOG_SETUP(".memory.logic.priority"); - -namespace storage::framework::defaultimplementation { - -PriorityMemoryLogic::PriorityMemoryLogic(Clock& c, uint64_t maxMem) - : SimpleMemoryLogic(c, maxMem) -{ - LOG(debug, "Setup priority memory logic with max memory of %lu bytes", maxMem); -} - -float -PriorityMemoryLogic::getNonCacheThreshold(uint8_t priority) const -{ - return 0.6 + ((255 - priority) / 255.0) * 0.4; -} - -void -PriorityMemoryLogic::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "PriorityMemoryLogic() : "; - SimpleMemoryLogic::print(out, verbose, indent); -} - -} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h deleted file mode 100644 index 148c4f40aee..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -/** - * \class storage::PriorityMemoryLogic - * - * \brief Priority logic deciding who should get memory and how much. - * - */ - -#pragma once - -#include "simplememorylogic.h" -#include <vespa/vespalib/util/sync.h> - -namespace storage { -namespace framework { -namespace defaultimplementation { - -struct PriorityMemoryLogic : public SimpleMemoryLogic -{ - PriorityMemoryLogic(Clock&, uint64_t maxMemory); - float getNonCacheThreshold(uint8_t priority) const override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} // defaultimplementation -} // framework -} // storage - diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp deleted file mode 100644 index 76d6f990a00..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simplememorylogic.h" -#include <vespa/vespalib/util/exceptions.h> - -#include <vespa/log/bufferedlogger.h> -LOG_SETUP(".memory.logic.simple"); - -namespace storage::framework::defaultimplementation { - -SimpleMemoryLogic::SimpleMemoryLogic(Clock& c, uint64_t maxMemory) - : _cacheThreshold(0.98), - _nonCacheThreshold(0.8), - _state(c, maxMemory), - _reducers() -{ - LOG(debug, "Setup simple memory logic with max memory of %" PRIu64 " bytes", maxMemory); -} - -SimpleMemoryLogic::~SimpleMemoryLogic() -{ -} - -void -SimpleMemoryLogic::setMaximumMemoryUsage(uint64_t max) -{ - vespalib::LockGuard lock(_stateLock); - _state.setMaximumMemoryUsage(max); -} - -void -SimpleMemoryLogic::getState(MemoryState& state, bool resetMax) { - vespalib::LockGuard lock(_stateLock); - state = _state; - if (resetMax) _state.resetMax(); -} - -MemoryToken::UP -SimpleMemoryLogic::allocate(const MemoryAllocationType& type, - uint8_t priority, - ReduceMemoryUsageInterface* reducer) -{ - MemoryTokenImpl::UP token( - new MemoryTokenImpl(*this, type, 0, priority, reducer)); - if (reducer != 0) { - vespalib::LockGuard lock(_stateLock); - _reducers.push_back(Reducer(*token, *reducer)); - } - return std::move(token); -} - -bool -SimpleMemoryLogic::resize(MemoryToken& tok, uint64_t min, uint64_t max, - uint32_t allocationCounts) -{ - vespalib::LockGuard lock(_stateLock); - MemoryTokenImpl& token(static_cast<MemoryTokenImpl&>(tok)); - LOG(spam, "Attempting to resize %s to size in the range %" PRIu64 " to " - "%" PRIu64 ".", token.toString().c_str(), min, max); - if (token.getSize() > max) { // Always safe to reduce size - handleReduction(token, max, allocationCounts); - return true; - } - // If not reducing size, calculate relative min/max values. - uint64_t relMin = (min > token.getSize() ? min - token.getSize() : 0); - uint64_t relMax = max - token.getSize(); - return resizeRelative(token, relMin, relMax, allocationCounts); -} - -void -SimpleMemoryLogic::handleReduction(MemoryTokenImpl& token, uint64_t max, - uint32_t allocationCounts) -{ - LOG(spam, "Reduzing size of token by %" PRIu64 ".", - token.getSize() - max); - _state.removeFromEntry(token.getType(), token.getSize() - max, - token.getPriority(), allocationCounts); - setTokenSize(token, max); -} - -bool -SimpleMemoryLogic::handleCacheMemoryRequest( - MemoryTokenImpl& token, uint64_t min, uint64_t max, - uint32_t allocationCounts) -{ - uint64_t usedSize(_state.getCurrentSnapshot().getUsedSize()); - uint64_t thresholdSize = uint64_t(getCacheThreshold() - * _state.getTotalSize()); - uint64_t toAllocate(thresholdSize > usedSize - ? std::min(thresholdSize - usedSize, max) - : 0); - bool forced = false; - if (token.getType().isAllocationsForced() && toAllocate < min) { - toAllocate = min; - forced = true; - } - if (toAllocate < min) { - LOG(spam, "We cannot give more memory to cache without going above " - "cache threshold (%" PRIu64 " B)", thresholdSize); - _state.addToEntry(token.getType(), 0, token.getPriority(), - MemoryState::DENIED, false, allocationCounts); - return false; - } - LOG(spam, "Giving %" PRIu64 " bytes of memory to cache. (Cache threshold " - "is %" PRIu64 ", used size is %" PRIu64 ", %" PRIu64 " bytes were " - "always allocated to the token and it wanted memory between %" - PRIu64 " and %" PRIu64 ".", - toAllocate, thresholdSize, usedSize, token.getSize(), min, max); - _state.addToEntry(token.getType(), toAllocate, token.getPriority(), - static_cast<uint64_t>(toAllocate) >= max - ? MemoryState::GOT_MAX : MemoryState::GOT_MIN, - forced, allocationCounts); - setTokenSize(token, token.getSize() + toAllocate); - return true; -} - -uint64_t -SimpleMemoryLogic::getMemorySizeFreeForPriority(uint8_t priority) const -{ - uint64_t usedSize(_state.getCurrentSnapshot().getUsedSizeIgnoringCache()); - uint64_t thresholdSize = uint64_t(getNonCacheThreshold(priority) - * _state.getTotalSize()); - return (usedSize >= thresholdSize ? 0 : thresholdSize - usedSize); -} - -bool -SimpleMemoryLogic::resizeRelative( - MemoryTokenImpl& token, uint64_t min, uint64_t max, - uint32_t allocationCounts) -{ - LOG(spam, "Relative resize change. Need another %zu-%zu byte of memory.", - min, max); - // If requester is cache, use cache threshold - if (token.getType().isCache()) { - return handleCacheMemoryRequest(token, min, max, allocationCounts); - } - // If we get here, requester is not cache. - uint64_t usedSize(_state.getCurrentSnapshot().getUsedSizeIgnoringCache()); - uint64_t thresholdSize = uint64_t(getNonCacheThreshold(token.getPriority()) - * _state.getTotalSize()); - uint64_t toAllocate = 0; - if (thresholdSize > usedSize) { - toAllocate = std::min(max, thresholdSize - usedSize); - } - if (toAllocate < min) toAllocate = min; - bool forced = false; - if (usedSize + toAllocate > _state.getTotalSize()) { - if (token.getType().isAllocationsForced()) { - forced = true; - } else { - LOG(spam, "We cannot give more memory without going beyond max"); - _state.addToEntry(token.getType(), 0, token.getPriority(), - MemoryState::DENIED, false, allocationCounts); - return false; - } - } - // External load should not fill up too much - if (usedSize + toAllocate > thresholdSize - && token.getType().isExternalLoad() - && !token.getType().isAllocationsForced()) - { - LOG(spam, "Not giving external load memory beyond threshold."); - _state.addToEntry(token.getType(), 0, token.getPriority(), - MemoryState::DENIED, false, allocationCounts); - return false; - } - // If this puts us above max with cache, remove some cache. - if (_state.getCurrentSnapshot().getUsedSize() + toAllocate - > _state.getTotalSize()) - { - uint64_t needed(_state.getCurrentSnapshot().getUsedSize() - + toAllocate - _state.getTotalSize()); - for (uint32_t i=0; i<_reducers.size(); ++i) { - MemoryTokenImpl& rtoken(*_reducers[i]._token); - uint64_t reduceBy(std::min(needed, rtoken.getSize())); - uint64_t reduced(_reducers[i]._reducer->reduceMemoryConsumption( - rtoken, reduceBy)); - _state.removeFromEntry(rtoken.getType(), reduced, - rtoken.getPriority(), 0); - setTokenSize(rtoken, rtoken.getSize() - reduced); - needed -= reduceBy; - if (needed == 0) break; - if (reduced < reduceBy) { - LOG(debug, "Reducer refused to free the full %" PRIu64 " bytes " - "requested. %" PRIu64 " bytes reduced in token %s.", - reduceBy, reduced, rtoken.toString().c_str()); - } - } - } - if (_state.getCurrentSnapshot().getUsedSize() + toAllocate - > _state.getTotalSize()) - { - LOGBP(debug, "Failed to free enough memory from cache. This puts us " - "above max memory."); - } - LOG(spam, "Giving %" PRIu64 " bytes of memory", toAllocate); - _state.addToEntry(token.getType(), toAllocate, token.getPriority(), - static_cast<uint64_t>(toAllocate) >= max - ? MemoryState::GOT_MAX : MemoryState::GOT_MIN, - forced, allocationCounts); - setTokenSize(token, token.getSize() + toAllocate); - return true; -} - -void -SimpleMemoryLogic::freeToken(MemoryTokenImpl& token) -{ - vespalib::LockGuard lock(_stateLock); - _state.removeFromEntry(token.getType(), token.getSize(), - token.getPriority(), token.getAllocationCount()); - if (token.getReducer() != 0) { - std::vector<Reducer> reducers; - reducers.reserve(_reducers.size() - 1); - for (uint32_t i=0; i<_reducers.size(); ++i) { - if (_reducers[i]._token != &token) reducers.push_back(_reducers[i]); - } - assert(reducers.size() + 1 == _reducers.size()); - reducers.swap(_reducers); - } -} - -void -SimpleMemoryLogic::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SimpleMemoryLogic() {\n" - << indent << " "; - vespalib::LockGuard lock(_stateLock); - _state.print(out, verbose, indent + " "); -} - -} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h deleted file mode 100644 index c394db40b29..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -/** - * \class storage::SimpleMemoryLogic - * - * \brief Simple logic deciding who should get memory and how much. - * - * There is a cache threshold. By default 98%. Cache will always get memory up - * till this fillrate. - * - * There is a non-cache threshold. Non-cache memory requesters will get maximum - * memory until threshold is reached. If getting maximum memory would go beyond - * the non-cache threshold, the requester will get enough memory to hit the - * threshold (if more than minimum), or get the minimum memory asked for, if - * that doesn't put usage above 100%. - * - * Usage above 100% is attempted avoided by freeing cache memory. If failing to - * free enough memory, request will fail, or minimum will be get if allocation - * is forced such that it cannot fail. In such a case, usage may go beyond 100%. - */ - -#pragma once - -#include "memorymanager.h" -#include "memorystate.h" -#include <vespa/vespalib/util/sync.h> - -namespace storage { -namespace framework { -namespace defaultimplementation { - -class SimpleMemoryLogic : public AllocationLogic -{ - float _cacheThreshold; - float _nonCacheThreshold; - vespalib::Lock _stateLock; - MemoryState _state; - struct Reducer { - MemoryTokenImpl* _token; - ReduceMemoryUsageInterface* _reducer; - - Reducer() : _token(0), _reducer(0) {} - Reducer(MemoryTokenImpl& t, - ReduceMemoryUsageInterface& r) - : _token(&t), _reducer(&r) {} - }; - std::vector<Reducer> _reducers; - -protected: - float getCacheThreshold() { return _cacheThreshold; } - - // Priority memory logic can override this to set a threshold based on - // priority - virtual float getNonCacheThreshold(uint8_t priority) const - { (void) priority; return _nonCacheThreshold; } - -public: - typedef std::unique_ptr<SimpleMemoryLogic> UP; - - SimpleMemoryLogic(Clock&, uint64_t maxMemory); - - ~SimpleMemoryLogic(); - - SimpleMemoryLogic& setMinJumpToUpdateMax(uint32_t bytes) { - _state.setMinJumpToUpdateMax(bytes); - return *this; - } - - void setMaximumMemoryUsage(uint64_t max) override; - - void setCacheThreshold(float limit) { _cacheThreshold = limit; } - void setNonCacheThreshold(float limit) { _nonCacheThreshold = limit; } - - MemoryState& getState() { return _state; } // Not threadsafe. Unit testing. - void getState(MemoryState& state, bool resetMax) override; - - MemoryToken::UP allocate(const MemoryAllocationType&, uint8_t priority, - ReduceMemoryUsageInterface* = 0) override; - bool resize(MemoryToken& token, uint64_t min, uint64_t max, uint32_t allocationCounts) override; - - void freeToken(MemoryTokenImpl& token) override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - virtual uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override; - -private: - void handleReduction(MemoryTokenImpl&, uint64_t size, - uint32_t allocationCounts); - bool resizeRelative(MemoryTokenImpl&, uint64_t min, uint64_t max, - uint32_t allocationCounts); - bool handleCacheMemoryRequest(MemoryTokenImpl&, uint64_t min, uint64_t max, - uint32_t allocationCounts); -}; - -} // defaultimplementation -} // framework -} // storage - diff --git a/storageframework/src/vespa/storageframework/generic/CMakeLists.txt b/storageframework/src/vespa/storageframework/generic/CMakeLists.txt index cf225f8d5aa..5df9963304d 100644 --- a/storageframework/src/vespa/storageframework/generic/CMakeLists.txt +++ b/storageframework/src/vespa/storageframework/generic/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_add_library(storageframework_generic $<TARGET_OBJECTS:storageframework_component> $<TARGET_OBJECTS:storageframework_status> $<TARGET_OBJECTS:storageframework_thread> - $<TARGET_OBJECTS:storageframework_memory> $<TARGET_OBJECTS:storageframework_clock> INSTALL lib64 DEPENDS diff --git a/storageframework/src/vespa/storageframework/generic/memory/.gitignore b/storageframework/src/vespa/storageframework/generic/memory/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/storageframework/src/vespa/storageframework/generic/memory/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt b/storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt deleted file mode 100644 index 77aa14e4ca2..00000000000 --- a/storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageframework_memory OBJECT - SOURCES - memorytoken.cpp - DEPENDS -) diff --git a/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h b/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h deleted file mode 100644 index e91b7946454..00000000000 --- a/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::framework::MemoryAllocationType - * \ingroup memory - * - * \brief Allocation types used to differ between memory manager clients. - * - * The different memory manager clients have different properties. It is - * important for the memory manager to distinguish between different users in - * order to know how to prioritize memory, and also in order to create good - * reports on memory usage. - * - * An allocation type holds metadata for a memory manager client, including a - * name for the type and various properties that may affect how much memory - * such a client will get, whether it always gets some, etc. - */ - -#pragma once - -#include <string> -#include <memory> - -namespace storage::framework { - -struct MemoryAllocationType { - using UP = std::unique_ptr<MemoryAllocationType>; - - enum Flags { - NONE = 0x00, - FORCE_ALLOCATE = 0x01, - EXTERNAL_LOAD = 0x02, - CACHE = 0x04 - }; - - MemoryAllocationType() - : _flags(NONE), _name("") {}; - - MemoryAllocationType(const std::string& name, uint32_t flags = NONE) - : _flags(flags), _name(name) {} - - const std::string& getName() const { return _name; } - bool isAllocationsForced() const { return (_flags & FORCE_ALLOCATE); } - bool isExternalLoad() const { return (_flags & EXTERNAL_LOAD); } - bool isCache() const { return (_flags & CACHE); } - -private: - uint32_t _flags; - std::string _name; -}; - -} diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h b/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h deleted file mode 100644 index a97038c2aab..00000000000 --- a/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::framework::MemoryManagerInterface - * \ingroup memory - * - * \brief Interface with functions clients need in order to use a memory manager - * - * This interface exist so clients can use a memory manager without actually - * depending on the implementation of it. - */ - -#pragma once - -#include "memoryallocationtype.h" -#include "memorytoken.h" -#include "reducememoryusageinterface.h" -#include <vector> - -namespace storage::framework { - -struct MemoryManagerInterface -{ - typedef std::unique_ptr<MemoryManagerInterface> UP; - - virtual ~MemoryManagerInterface() {} - - virtual void setMaximumMemoryUsage(uint64_t max) = 0; - - /** - * Registers the given allocation type by copying it, and returning - * a reference to the copied object. - */ - virtual const MemoryAllocationType& - registerAllocationType(const MemoryAllocationType& type) = 0; - - /** Throws exception if failing to find type. */ - virtual const MemoryAllocationType& - getAllocationType(const std::string& name) const = 0; - - /** Get an overview of all registration types. */ - virtual std::vector<const MemoryAllocationType*> - getAllocationTypes() const = 0; - - /** - * Decide how much to allocate for this request. Should be between min - * and max, unless it's of a type that can be denied (such as external - * requests), in which case we can also deny allocation by returning a null - * token. - */ - virtual MemoryToken::UP allocate( - const MemoryAllocationType&, - uint64_t min, - uint64_t max, - uint8_t priority, - ReduceMemoryUsageInterface* = 0) = 0; - - /** - * Utility function to see how much memory is available. - */ - virtual uint64_t getMemorySizeFreeForPriority(uint8_t priority) const = 0; -}; - -} diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp b/storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp deleted file mode 100644 index 3950810a010..00000000000 --- a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memorytoken.h" - -namespace storage { -namespace framework { - -MemoryToken::~MemoryToken() -{ -} - -} // framework -} // storage diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h b/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h deleted file mode 100644 index cd389d64549..00000000000 --- a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::framework::MemoryToken - * \ingroup memory - * - * \brief Token to keep by client for current allocations. - * - * This class is a token a memory manager client will get from the memory - * manager when getting memory. It can be used to know how much you currently - * have allocated, and through it you can request more or less memory. - */ - -#pragma once - -#include <memory> - -namespace storage::framework { - -class MemoryToken { -protected: -public: - typedef std::unique_ptr<MemoryToken> UP; - virtual ~MemoryToken(); - - virtual uint64_t getSize() const = 0; - virtual bool resize(uint64_t min, uint64_t max) = 0; -}; - -} diff --git a/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h b/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h deleted file mode 100644 index cd50d33015c..00000000000 --- a/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::framework::ReduceMemoryUsageInterface - * \ingroup memory - * - * \brief The manager can take memory back when needed using this interface. - * - * Some memory users, typically caches, wants to use all available memory. But - * to let them use all available memory, it must also be easy to take memory - * back when needed for something else. An implementation of this interface can - * be given on memory allocations to give the memory manager the ability to take - * memory back when needed. - */ - -#pragma once - -namespace storage::framework { - -struct ReduceMemoryUsageInterface -{ - virtual ~ReduceMemoryUsageInterface() {} - - /** - * This callback is called when the memory manager want to reduce the usage - * of the given memory token. Actual memory to be released should be - * released in this function. The token itself will be adjusted by the - * memory manager though. The memory manager may keep a lock through this - * call, so no memory manager calls should be made inside this callback. - * - * It is recommended that you actually release at least as many bytes as - * requested. Though currently it is allowed to reduce less or refuse, but - * this might mean that some higher priority task does not get the memory it - * needs. - * - * @param reduceBy Always in the range 0 < reduceBy <= token.size() - * @return The amount of memory no longer used. - */ - virtual uint64_t reduceMemoryConsumption(const MemoryToken&, uint64_t reduceBy) = 0; -}; - -} diff --git a/storageframework/src/vespa/storageframework/storageframework.h b/storageframework/src/vespa/storageframework/storageframework.h index f9847d5fd12..4ab36fa099f 100644 --- a/storageframework/src/vespa/storageframework/storageframework.h +++ b/storageframework/src/vespa/storageframework/storageframework.h @@ -7,7 +7,6 @@ #include <vespa/storageframework/generic/clock/clock.h> #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageframework/generic/component/component.h> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageframework/generic/status/statusreportermap.h> |