diff options
Diffstat (limited to 'storage/src')
36 files changed, 21 insertions, 1373 deletions
diff --git a/storage/src/tests/CMakeLists.txt b/storage/src/tests/CMakeLists.txt index 7747e8bacc1..9962949edde 100644 --- a/storage/src/tests/CMakeLists.txt +++ b/storage/src/tests/CMakeLists.txt @@ -13,7 +13,6 @@ vespa_add_executable(storage_testrunner_app TEST storage_testdistributor storage_testpersistence storage_testfilestorage - storage_testmemory storage_teststatus ) diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index 6906d41ac47..46744ecb3c6 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -24,7 +24,6 @@ #include <vespa/storage/storageserver/framework.h> #include <vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h> #include <vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storageframework/defaultimplementation/component/testcomponentregister.h> #include <vespa/persistence/spi/persistenceprovider.h> diff --git a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt b/storage/src/tests/frameworkimpl/memory/CMakeLists.txt deleted file mode 100644 index e5a364e02a3..00000000000 --- a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testmemory TEST - SOURCES - memorystatusviewertest.cpp - DEPENDS - storage - storage_testcommon -) diff --git a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp b/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp deleted file mode 100644 index 1ad4b16751d..00000000000 --- a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/metrics/metrics.h> -#include <vespa/metrics/metricmanager.h> -#include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h> -#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h> -#include <tests/common/teststorageapp.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/vespalib/util/exceptions.h> -#include <boost/lexical_cast.hpp> - -namespace storage { - -struct MemoryStatusViewerTest : public CppUnit::TestFixture -{ - static const int maxMemory = 1000; - std::unique_ptr<TestServiceLayerApp> _node; - std::unique_ptr<framework::defaultimplementation::MemoryManager> _memMan; - - void setUp() override; - - void testEmptyState(); - void testSnapshots(); - - CPPUNIT_TEST_SUITE(MemoryStatusViewerTest); - CPPUNIT_TEST(testEmptyState); - CPPUNIT_TEST(testSnapshots); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemoryStatusViewerTest); - -void -MemoryStatusViewerTest::setUp() -{ - _node.reset(new TestServiceLayerApp(DiskCount(2))); - framework::defaultimplementation::PriorityMemoryLogic* logic( - new framework::defaultimplementation::PriorityMemoryLogic( - _node->getClock(), maxMemory)); - logic->setMinJumpToUpdateMax(1); - _memMan.reset(new framework::defaultimplementation::MemoryManager( - framework::defaultimplementation::AllocationLogic::UP(logic))); -} - -void -MemoryStatusViewerTest::testEmptyState() -{ - // Add a memory manager, and add a bit of load to it, so it's not - // totally empty. - StorageComponent component(_node->getComponentRegister(), "test"); - - metrics::MetricManager mm; - MemoryStatusViewer viewer( - *_memMan, mm, _node->getComponentRegister()); - std::ostringstream actual; - viewer.reportStatus(actual, framework::HttpUrlPath("/")); - CPPUNIT_ASSERT_MATCH_REGEX(".*Plotr.LineChart.*", actual.str()); - CPPUNIT_ASSERT_MATCH_REGEX( - ".*Current: 1970-01-01 00:00:00 Max memory 1000 SnapShot\\(Used 0, w/o cache 0\\).*", - actual.str()); - CPPUNIT_ASSERT_MATCH_REGEX( - ".*Last hour: na.*", actual.str()); -} - -namespace { - void waitForProcessedTime( - const MemoryStatusViewer& viewer, framework::SecondTime time, - framework::SecondTime timeout = framework::SecondTime(30)) - { - framework::defaultimplementation::RealClock clock; - framework::MilliSecTime endTime( - clock.getTimeInMillis() + timeout.getMillis()); - framework::SecondTime processedTime(0); - while (clock.getTimeInMillis() < endTime) { - processedTime = viewer.getProcessedTime(); - if (processedTime >= time) return; - FastOS_Thread::Sleep(1); - } - std::ostringstream ost; - ost << "Timed out waiting " << timeout << " ms for time " << time - << " to be processed. Currently time is only processed up to " - << processedTime; - throw new vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); - } -} - -#define ASSERT_MEMORY(output, period, maxmem, used, usedwocache) \ -{ \ - std::string::size_type _pos1_(output.find(period)); \ - std::string::size_type _pos2_(output.find("Max memory", _pos1_)); \ - std::string::size_type _pos3_(output.find("SnapShot", _pos2_)); \ - std::string _maxMemory_(output.substr(_pos2_ + 11, _pos3_ - _pos2_ - 12)); \ - std::string::size_type _pos4_(output.find(",", _pos3_)); \ - std::string _used_(output.substr(_pos3_ + 14, _pos4_ - _pos3_ - 14)); \ - std::string::size_type _pos5_(output.find(")", _pos4_)); \ - std::string _usedwo_(output.substr(_pos4_ + 12, _pos5_ - _pos4_ - 12)); \ - std::ostringstream _failure_; \ - _failure_ << "Wrong match in period " << period << " in output:\n" \ - << output << "\nFor value: "; \ - \ - CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Max memory", \ - uint64_t(maxmem), boost::lexical_cast<uint64_t>(_maxMemory_)); \ - CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Used memory", \ - uint64_t(used), boost::lexical_cast<uint64_t>(_used_)); \ - CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Used memory w/o cache", \ - uint64_t(usedwocache), boost::lexical_cast<uint64_t>(_usedwo_)); \ -} - -void -MemoryStatusViewerTest::testSnapshots() -{ - // Add a memory manager, and add a bit of load to it, so it's not - // totally empty. - StorageComponent component(_node->getComponentRegister(), "test"); - const framework::MemoryAllocationType putAlloc( - component.getMemoryManager().registerAllocationType( - framework::MemoryAllocationType("PUT"))); - const framework::MemoryAllocationType getAlloc( - component.getMemoryManager().registerAllocationType( - framework::MemoryAllocationType("GET"))); - - framework::MemoryToken::UP put = _memMan->allocate(putAlloc, 0, 100, 80); - framework::MemoryToken::UP get = _memMan->allocate(getAlloc, 30, 200, 50); - framework::MemoryToken::UP get2 = _memMan->allocate(getAlloc, 70, 150, 60); - - metrics::MetricManager mm; - MemoryStatusViewer viewer(*_memMan, mm, _node->getComponentRegister()); - - _node->getClock().addSecondsToTime(1000); - viewer.notifyThread(); - waitForProcessedTime(viewer, framework::SecondTime(1000)); - - std::ostringstream actual; - viewer.printDebugOutput(actual); - //std::cerr << actual.str() << "\n"; - ASSERT_MEMORY(actual.str(), "Current", 1000, 450, 450); - ASSERT_MEMORY(actual.str(), "Last hour", 1000, 450, 450); - ASSERT_MEMORY(actual.str(), "Last ever", 1000, 450, 450); - - put = _memMan->allocate(putAlloc, 0, 50, 80); - get = _memMan->allocate(getAlloc, 100, 140, 50); - get2 = _memMan->allocate(getAlloc, 20, 100, 70); - - _node->getClock().addSecondsToTime(3600); - viewer.notifyThread(); - waitForProcessedTime(viewer, framework::SecondTime(4600)); - - actual.str(""); - viewer.printDebugOutput(actual); - //std::cerr << actual.str() << "\n"; - ASSERT_MEMORY(actual.str(), "Current", 1000, 290, 290); - ASSERT_MEMORY(actual.str(), "Last hour", 1000, 540, 540); - ASSERT_MEMORY(actual.str(), "Last ever", 1000, 540, 540); - - get.reset(); - - _node->getClock().addSecondsToTime(3600); - viewer.notifyThread(); - waitForProcessedTime(viewer, framework::SecondTime(4600 + 3600)); - - actual.str(""); - viewer.printDebugOutput(actual); - //std::cerr << actual.str() << "\n"; - ASSERT_MEMORY(actual.str(), "Current", 1000, 150, 150); - ASSERT_MEMORY(actual.str(), "Last hour", 1000, 290, 290); - ASSERT_MEMORY(actual.str(), "Last ever", 1000, 540, 540); - -} - -} // storage diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 67b93108139..af0082fa788 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -231,8 +231,6 @@ struct FileStorManagerTest : public CppUnit::TestFixture { fprintf(stderr, "%s\n", e.what()); } _testdoctype1 = _node->getTypeRepo()->getDocumentType("testdoctype1"); - _node->getMemoryManager().registerAllocationType( - framework::MemoryAllocationType("VISITOR_BUFFER")); } void putDoc(DummyStorageLink& top, @@ -2010,19 +2008,11 @@ FileStorManagerTest::testVisiting() top.reset(); // Visit bucket with no split, using no selection { - framework::MemoryToken::UP token( - _node->getMemoryManager().allocate( - _node->getMemoryManager().getAllocationType( - "VISITOR_BUFFER"), - 16*1024, - 16*1024, - 127)); spi::IteratorId iterId(createIterator(top, ids[0], "true")); - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), makeDocumentBucket(ids[0]), iterId, 16*1024)); + auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[0]), iterId, 16*1024); top.sendDown(cmd); top.waitForMessages(1, _waitTime); - CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies()); std::shared_ptr<GetIterReply> reply( std::dynamic_pointer_cast<GetIterReply>(top.getReply(0))); CPPUNIT_ASSERT(reply.get()); @@ -2039,15 +2029,7 @@ FileStorManagerTest::testVisiting() ids[1], "testdoctype1.hstringval = \"John Doe\"")); while (true) { - framework::MemoryToken::UP token( - _node->getMemoryManager().allocate( - _node->getMemoryManager().getAllocationType( - "VISITOR_BUFFER"), - 16*1024, - 16*1024, - 127)); - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), makeDocumentBucket(ids[1]), iterId, 16*1024)); + auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[1]), iterId, 16*1024); top.sendDown(cmd); top.waitForMessages(1, _waitTime); CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); @@ -2078,18 +2060,10 @@ FileStorManagerTest::testVisiting() true)); uint32_t totalDocs = 0; while (true) { - framework::MemoryToken::UP token( - _node->getMemoryManager().allocate( - _node->getMemoryManager().getAllocationType( - "VISITOR_BUFFER"), - 16*1024, - 16*1024, - 127)); - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), makeDocumentBucket(ids[1]), iterId, 16*1024)); + auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[1]), iterId, 16*1024); top.sendDown(cmd); top.waitForMessages(1, _waitTime); - CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies()); std::shared_ptr<GetIterReply> reply( std::dynamic_pointer_cast<GetIterReply>( top.getReply(0))); @@ -2799,15 +2773,8 @@ FileStorManagerTest::testGetIter() // Sending a getiter request that will only visit some of the docs spi::IteratorId iterId(createIterator(top, bid, "")); { - framework::MemoryToken::UP token( - _node->getMemoryManager().allocate( - _node->getMemoryManager().getAllocationType( - "VISITOR_BUFFER"), - 2048, - 2048, - 127)); std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), makeDocumentBucket(bid), iterId, 2048)); + new GetIterCommand(makeDocumentBucket(bid), iterId, 2048)); top.sendDown(cmd); top.waitForMessages(1, _waitTime); CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); @@ -2837,18 +2804,10 @@ FileStorManagerTest::testGetIter() CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); } { - framework::MemoryToken::UP token( - _node->getMemoryManager().allocate( - _node->getMemoryManager().getAllocationType( - "VISITOR_BUFFER"), - 2048, - 2048, - 127)); - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), makeDocumentBucket(bid), iterId, 2048)); + auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(bid), iterId, 2048); top.sendDown(cmd); top.waitForMessages(1, _waitTime); - CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies()); std::shared_ptr<GetIterReply> reply( std::dynamic_pointer_cast<GetIterReply>( top.getReply(0))); diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index fcfbcd0a78b..9d3a3d5f008 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -11,7 +11,6 @@ #include <vespa/vespalib/util/exceptions.h> using document::DocumentType; -using storage::framework::defaultimplementation::AllocationLogic; using storage::spi::test::makeSpiBucket; using document::test::makeDocumentBucket; diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h index ee87925a0bc..36b1fef7285 100644 --- a/storage/src/tests/persistence/persistencetestutils.h +++ b/storage/src/tests/persistence/persistencetestutils.h @@ -8,7 +8,6 @@ #include <vespa/storage/persistence/persistenceutil.h> #include <vespa/storage/common/messagesender.h> #include <vespa/storage/common/storagecomponent.h> -#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/document/base/testdocman.h> diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp index 5111368dd1c..cd5dd2a01a4 100644 --- a/storage/src/tests/storageserver/communicationmanagertest.cpp +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -6,7 +6,6 @@ #include <vespa/messagebus/rpcmessagebus.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> -#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h> #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> #include <tests/common/testhelper.h> diff --git a/storage/src/vespa/storage/CMakeLists.txt b/storage/src/vespa/storage/CMakeLists.txt index 69846cafde2..d854c603097 100644 --- a/storage/src/vespa/storage/CMakeLists.txt +++ b/storage/src/vespa/storage/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_library(storage $<TARGET_OBJECTS:storage_bucketmover> $<TARGET_OBJECTS:storage_thread> $<TARGET_OBJECTS:storage_status> - $<TARGET_OBJECTS:storage_memory> $<TARGET_OBJECTS:storage_component> INSTALL lib64 DEPENDS diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 677ca0a6309..aa9158f9e5b 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -34,7 +34,6 @@ BucketManager::BucketManager(const config::ConfigUri & configUri, : StorageLinkQueued("Bucket manager", compReg), framework::StatusReporter("bucketdb", "Bucket database"), _configUri(configUri), - _bucketDBMemoryToken(), _workerLock(), _workerCond(), _clusterStateLock(), @@ -48,12 +47,6 @@ BucketManager::BucketManager(const config::ConfigUri & configUri, _requestsCurrentlyProcessing(0), _component(compReg, "bucketmanager") { - const framework::MemoryAllocationType& allocType( - _component.getMemoryManager().registerAllocationType( - framework::MemoryAllocationType("DATABASE"))); - _bucketDBMemoryToken = _component.getMemoryManager().allocate( - allocType, 0, 0, api::StorageMessage::HIGH); - assert(_bucketDBMemoryToken.get() != 0); _metrics->setDisks(_component.getDiskCount()); _component.registerStatusPage(*this); _component.registerMetric(*_metrics); @@ -222,8 +215,6 @@ BucketManager::updateMetrics(bool updateDocCount) LOG(debug, "Iterating bucket database to update metrics%s%s", updateDocCount ? "" : ", minusedbits only", _doneInitialized ? "" : ", server is not done initializing"); - uint64_t dbMemSize = _component.getBucketSpaceRepo().getBucketMemoryUsage(); - _bucketDBMemoryToken->resize(dbMemSize, dbMemSize); uint32_t diskCount = _component.getDiskCount(); if (!updateDocCount || _doneInitialized) { diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index 5e4e928dfc7..8413c5a0f42 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -20,7 +20,6 @@ #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/common/storagelinkqueued.h> #include <vespa/storageapi/message/bucket.h> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/storageframework/generic/status/statusreporter.h> @@ -47,7 +46,6 @@ private: config::ConfigUri _configUri; uint32_t _chunkLevel; - framework::MemoryToken::UP _bucketDBMemoryToken; BucketInfoRequestMap _bucketInfoRequests; /** diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp index de44116b316..d458a821e02 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp +++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "statusmetricconsumer.h" -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <boost/assign.hpp> #include <boost/lexical_cast.hpp> @@ -26,17 +25,8 @@ StatusMetricConsumer::StatusMetricConsumer( _component(compReg, "statusmetricsconsumer"), _name(name), _startTime(_component.getClock().getTimeInSeconds()), - _processedTime(0), - _metricMemoryToken() + _processedTime(0) { - const framework::MemoryAllocationType& allocType( - _component.getMemoryManager().registerAllocationType( - framework::MemoryAllocationType( - "METRICS", framework::MemoryAllocationType::FORCE_ALLOCATE) - )); - _metricMemoryToken = _component.getMemoryManager().allocate( - allocType, 0, 0, api::StorageMessage::HIGH); - assert(_metricMemoryToken.get() != 0); LOG(debug, "Started metrics consumer"); setlocale(LC_NUMERIC, ""); _component.registerMetricUpdateHook(*this, framework::SecondTime(3600)); @@ -51,8 +41,7 @@ void StatusMetricConsumer::updateMetrics(const MetricLockGuard & guard) { metrics::MemoryConsumption::UP mc(_manager.getMemoryConsumption(guard)); - uint32_t usage = mc->getTotalMemoryUsage(); - _metricMemoryToken->resize(usage, usage); + // TODO is this hook needed anymore? } vespalib::string diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.h b/storage/src/vespa/storage/common/statusmetricconsumer.h index e3bae51cf99..6f93f51cfdf 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.h +++ b/storage/src/vespa/storage/common/statusmetricconsumer.h @@ -53,7 +53,6 @@ private: vespalib::Monitor _waiter; framework::SecondTime _startTime; framework::SecondTime _processedTime; - std::unique_ptr<framework::MemoryToken> _metricMemoryToken; void writeXmlTags(std::ostream& out, const vespalib::StringTokenizer& name, diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index a3c9804c2b4..29e8d3f6221 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -13,7 +13,6 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/common/storagelink.h> #include <vespa/storageframework/generic/clock/timer.h> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageapi/messageapi/messagehandler.h> #include <set> #include <deque> diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h index e8480902549..8d31a228543 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h @@ -6,7 +6,6 @@ #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/storageapi/messageapi/maintenancecommand.h> #include <vespa/document/bucket/bucketid.h> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> namespace storage::distributor { @@ -229,7 +228,6 @@ protected: bool _ok; api::StorageMessage::Priority _priority; - framework::MemoryToken::UP _memoryToken; /** * Checks if the given bucket is blocked by any pending messages to any diff --git a/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt b/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt deleted file mode 100644 index 2a91ead2b60..00000000000 --- a/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_memory OBJECT - SOURCES - memorystatusviewer.cpp - DEPENDS - AFTER - storage_storageconfig -) diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h b/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h deleted file mode 100644 index fd47bec1424..00000000000 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::MemorySnapshotList - * - * \brief Holds a historic list of MemoryStates. - * - */ - -#pragma once - -#include <storage/frameworkimpl/memory/memorystate.h> - -namespace storage { - -class MemorySnapshotList : public vespalib::Printable { - std::map<uint64_t time, MemoryState> _snapshots; -}; - -} // storage - diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp deleted file mode 100644 index dd07a61f27b..00000000000 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp +++ /dev/null @@ -1,661 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memorystatusviewer.h" -#include <vespa/storage/storageutil/graph.h> -#include <vespa/storage/storageutil/palette.h> -#include <vespa/storage/storageutil/piechart.h> -#include <vespa/metrics/metricmanager.h> -#include <vespa/storageapi/messageapi/storagemessage.h> -#include <algorithm> - -#include <vespa/log/bufferedlogger.h> -LOG_SETUP(".memory.status.viewer"); - -using storage::framework::defaultimplementation::MemoryState; - -namespace storage { - -MemoryStatusViewer::Entry::Entry( - const std::string& name, framework::Clock& clock, - framework::SecondTime maxAge) - : _name(name), - _maxAge(maxAge), - _timeTaken(clock.getTimeInSeconds()), - _data(), - _maxMemory(0) -{ -} - -MemoryStatusViewer::MemoryStatusViewer( - framework::defaultimplementation::MemoryManager& mm, - const metrics::MetricManager& metricMan, - StorageComponentRegister& compReg) - : framework::HtmlStatusReporter("memorymanager", "Memory Manager"), - _component(compReg, "memorystatusviewer"), - _manager(mm), - _metricManager(metricMan), - _workerMonitor(), - _states(), - _memoryHistory(), - _memoryHistorySize(24 * 31), - _memoryHistoryPeriod(60), - _allowedSlackPeriod(6), - _lastHistoryUpdate(_component.getClock().getTimeInSeconds()), - _processedTime(0) -{ - addEntry("Current", 0); - addEntry("Last hour", 60 * 60); - addEntry("Last day", 24 * 60 * 60); - addEntry("Last month", 4 * 7 * 24 * 60 * 60); - addEntry("Last ever", std::numeric_limits<uint32_t>::max()); - - framework::MilliSecTime maxProcessingTime(60 * 1000); - _thread = _component.startThread(*this, maxProcessingTime, - framework::MilliSecTime(1000)); - _component.registerStatusPage(*this); -} - -MemoryStatusViewer::~MemoryStatusViewer() -{ - if (_thread.get() != 0) { - _thread->interrupt(); - { - vespalib::MonitorGuard monitor(_workerMonitor); - monitor.signal(); - } - _thread->join(); - } -} - -namespace { - struct Group { - std::set<const framework::MemoryAllocationType*> types; - api::StorageMessage::Priority minPri; - api::StorageMessage::Priority maxPri; - MemoryState::Entry entry; - - Group(const framework::MemoryAllocationType& type, - api::StorageMessage::Priority pri, - const MemoryState::Entry& e) - : types(), - minPri(pri), - maxPri(pri), - entry(e) - { - types.insert(&type); - } - }; - - struct GroupSizeOrder { - bool operator()(const Group& g1, const Group& g2) { - return (g1.entry._currentUsedSize > g2.entry._currentUsedSize); - } - }; - struct GroupAllocsOrder { - bool operator()(const Group& g1, const Group& g2) { - return (g1.entry._totalUserCount > g2.entry._totalUserCount); - } - }; - struct GroupMinAllocsOrder { - bool operator()(const Group& g1, const Group& g2) { - return (g1.entry._minimumCount > g2.entry._minimumCount); - } - }; - struct GroupDeniedAllocsOrder { - bool operator()(const Group& g1, const Group& g2) { - return (g1.entry._deniedCount > g2.entry._deniedCount); - } - }; - - std::vector<Group> collapsePriorities(MemoryStatusViewer::Entry& entry) { - std::vector<Group> groups; - const MemoryState::SnapShot& ss(entry._data); - for (MemoryState::AllocationMap::const_iterator it - = ss.getAllocations().begin(); - it != ss.getAllocations().end(); ++it) - { - std::unique_ptr<Group> group; - for (MemoryState::PriorityMap::const_iterator it2 - = it->second.begin(); it2 != it->second.end(); ++it2) - { - if (group.get() == 0) { - group.reset(new Group( - *it->first, it2->first, it2->second)); - } else { - group->entry += it2->second; - group->minPri = std::min(group->minPri, it2->first); - group->maxPri = std::max(group->maxPri, it2->first); - } - } - if (group.get() != 0) { - groups.push_back(*group); - } - } - return groups; - } - - std::vector<Group> groupLoad(uint32_t groupCount, uint64_t minSize, - uint32_t minEntries, - MemoryStatusViewer::Entry& entry) - { - assert(groupCount > 1); - std::vector<Group> groups(collapsePriorities(entry)); - if (groups.size() == 0) return groups; - std::sort(groups.begin(), groups.end(), GroupSizeOrder()); - assert(groups.front().entry._currentUsedSize - >= groups.back().entry._currentUsedSize); - while (groups.size() > minEntries - && (groups.size() > groupCount - || groups[groups.size() - 2].entry._currentUsedSize - < minSize)) - { - Group& nextButLast(groups[groups.size() - 2]); - Group& last(groups.back()); - if (last.entry._currentUsedSize > 0) { - nextButLast.entry += last.entry; - nextButLast.minPri = std::min(nextButLast.minPri, last.minPri); - nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri); - nextButLast.types.insert(*last.types.begin()); - } - groups.pop_back(); - } - return groups; - } - - std::vector<Group> groupAllocs(uint32_t groupCount, uint64_t minSize, - uint32_t minEntries, - MemoryStatusViewer::Entry& entry) - { - assert(groupCount > 1); - std::vector<Group> groups(collapsePriorities(entry)); - if (groups.size() == 0) return groups; - std::sort(groups.begin(), groups.end(), GroupAllocsOrder()); - assert(groups.front().entry._totalUserCount - >= groups.back().entry._totalUserCount); - while (groups.size() > minEntries - && (groups.size() > groupCount - || groups[groups.size() - 2].entry._totalUserCount - < minSize)) - { - Group& nextButLast(groups[groups.size() - 2]); - Group& last(groups.back()); - nextButLast.entry += last.entry; - nextButLast.minPri = std::min(nextButLast.minPri, last.minPri); - nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri); - nextButLast.types.insert(*last.types.begin()); - groups.pop_back(); - } - return groups; - } - - std::vector<Group> groupMinAllocs(uint32_t groupCount, uint64_t minSize, - uint32_t minEntries, - MemoryStatusViewer::Entry& entry) - { - assert(groupCount > 1); - std::vector<Group> groups(collapsePriorities(entry)); - if (groups.size() == 0) return groups; - std::sort(groups.begin(), groups.end(), GroupMinAllocsOrder()); - assert(groups.front().entry._minimumCount - >= groups.back().entry._minimumCount); - while (groups.size() > minEntries - && (groups.size() > groupCount - || groups[groups.size() - 2].entry._minimumCount - < minSize)) - { - Group& nextButLast(groups[groups.size() - 2]); - Group& last(groups.back()); - nextButLast.entry += last.entry; - nextButLast.minPri = std::min(nextButLast.minPri, last.minPri); - nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri); - nextButLast.types.insert(*last.types.begin()); - groups.pop_back(); - } - return groups; - } - - std::vector<Group> groupDeniedAllocs(uint32_t groupCount, uint64_t minSize, - uint32_t minEntries, - MemoryStatusViewer::Entry& entry) - { - assert(groupCount > 1); - std::vector<Group> groups(collapsePriorities(entry)); - if (groups.size() == 0) return groups; - std::sort(groups.begin(), groups.end(), GroupDeniedAllocsOrder()); - assert(groups.front().entry._deniedCount - >= groups.back().entry._deniedCount); - while (groups.size() > minEntries - && (groups.size() > groupCount - || groups[groups.size() - 2].entry._deniedCount - < minSize)) - { - Group& nextButLast(groups[groups.size() - 2]); - Group& last(groups.back()); - nextButLast.entry += last.entry; - nextButLast.minPri = std::min(nextButLast.minPri, last.minPri); - nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri); - nextButLast.types.insert(*last.types.begin()); - groups.pop_back(); - } - return groups; - } -} - -void -MemoryStatusViewer::printSnapshot( - std::ostream& out, Entry& entry, - std::map<const framework::MemoryAllocationType*, - uint32_t>& colors) const -{ - out << "<h4>" << entry._name << " - Taken at " - << entry._timeTaken.toString() << "</h4>\n" - << "<table><tr><td>\n" - << "<b>Memory usage"; - if (entry._name != "Current") { - out << ", maxed at " << framework::SecondTime(entry._timeTaken); - } - out << " with " - << (entry._data.getUsedSizeIgnoringCache() / (1024 * 1024)) - << " MB.</b><br>\n"; - std::string piename = entry._name; - std::replace(piename.begin(), piename.end(), ' ', '_'); - uint64_t freeSize = entry._maxMemory - entry._data.getUsedSize(); - // Memory usage pie - uint64_t minSize = freeSize / 20; - std::vector<Group> groups(groupLoad(20, minSize, 5, entry)); - PieChart chart(piename, PieChart::SCHEME_CUSTOM); - chart.printLabels(false); - for (uint32_t i=0; i<groups.size(); ++i) { - std::string name = "Other"; - if (groups[i].types.size() == 1) { - name = (*groups[i].types.begin())->getName(); - } - uint32_t mbytes = groups[i].entry._currentUsedSize / (1024 * 1024); - std::ostringstream ost; - ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri); - if (groups[i].minPri != groups[i].maxPri) { - ost << " - " << static_cast<uint16_t>(groups[i].maxPri); - } - ost << " (" << mbytes << " MB)"; - name = ost.str(); - if (groups[i].entry._currentUsedSize > 0) { - chart.add(groups[i].entry._currentUsedSize, name, - colors[*groups[i].types.begin()]); - } - } - { - std::ostringstream ost; - ost << "Free (" << (freeSize / (1024 * 1024)) << " MB)"; - chart.add(freeSize, ost.str(), colors[0]); - } - chart.printCanvas(out, 750, 300); - out << "\n\n"; - chart.printScript(out, ""); - out << "\n\n"; - // Total allocations pie - out << "</td><td>\n"; - PieChart allocChart(piename + "Alloc", PieChart::SCHEME_CUSTOM); - allocChart.printLabels(false); - groups = groupAllocs(20, 100, 5, entry); - uint64_t totalAllocs = 0; - for (uint32_t i=0; i<groups.size(); ++i) { - std::string name = "Other"; - if (groups[i].types.size() == 1) { - name = (*groups[i].types.begin())->getName(); - } - uint32_t allocs = groups[i].entry._totalUserCount; - totalAllocs += allocs; - std::ostringstream ost; - ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri); - if (groups[i].minPri != groups[i].maxPri) { - ost << " - " << static_cast<uint16_t>(groups[i].maxPri); - } - ost << " (" << allocs << " allocations)"; - name = ost.str(); - if (groups[i].entry._totalUserCount > 0) { - allocChart.add(groups[i].entry._totalUserCount, name, - colors[*groups[i].types.begin()]); - } - } - out << "<b>Allocations, totalling " << totalAllocs << "</b><br>\n"; - allocChart.printCanvas(out, 750, 300); - out << "\n\n"; - allocChart.printScript(out, ""); - out << "\n\n"; - out << "</td></tr><tr><td>\n"; - PieChart minChart(piename + "Min", PieChart::SCHEME_CUSTOM); - minChart.printLabels(false); - groups = groupMinAllocs(20, 100, 5, entry); - uint64_t totalMinAllocs = 0; - for (uint32_t i=0; i<groups.size(); ++i) { - std::string name = "Other"; - if (groups[i].types.size() == 1) { - name = (*groups[i].types.begin())->getName(); - } - uint32_t allocs = groups[i].entry._minimumCount; - totalMinAllocs += allocs; - std::ostringstream ost; - ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri); - if (groups[i].minPri != groups[i].maxPri) { - ost << " - " << static_cast<uint16_t>(groups[i].maxPri); - } - ost << " (" << allocs << " min allocations)"; - name = ost.str(); - if (groups[i].entry._minimumCount > 0) { - minChart.add(groups[i].entry._minimumCount, name, - colors[*groups[i].types.begin()]); - } - } - out << "<b>Minimum allocations, totalling " << totalMinAllocs - << "</b><br>\n"; - if (totalMinAllocs > 0) { - minChart.printCanvas(out, 750, 300); - out << "\n\n"; - minChart.printScript(out, ""); - out << "\n\n"; - } - out << "</td><td>\n"; - PieChart deniedChart(piename + "Denied", PieChart::SCHEME_CUSTOM); - deniedChart.printLabels(false); - groups = groupDeniedAllocs(20, 100, 5, entry); - uint64_t totalDeniedAllocs = 0; - for (uint32_t i=0; i<groups.size(); ++i) { - std::string name = "Other"; - if (groups[i].types.size() == 1) { - name = (*groups[i].types.begin())->getName(); - } - uint32_t allocs = groups[i].entry._deniedCount; - totalDeniedAllocs += allocs; - std::ostringstream ost; - ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri); - if (groups[i].minPri != groups[i].maxPri) { - ost << " - " << static_cast<uint16_t>(groups[i].maxPri); - } - ost << " (" << allocs << " denied allocations)"; - name = ost.str(); - if (groups[i].entry._deniedCount > 0) { - deniedChart.add(groups[i].entry._deniedCount, name, - colors[*groups[i].types.begin()]); - } - } - out << "<b>Denied allocations, totalling " << totalDeniedAllocs - << "</b><br>\n"; - if (totalDeniedAllocs > 0) { - deniedChart.printCanvas(out, 750, 300); - out << "\n\n"; - deniedChart.printScript(out, ""); - out << "\n\n"; - } - out << "</td></tr></table>\n"; -} - -void -MemoryStatusViewer::reportHtmlHeaderAdditions( - std::ostream& out, const framework::HttpUrlPath&) const -{ - (void) out; - // FIXME this function used to emit Yahoo-internal links to graph plotting - // JS files. Obviously, this won't work for external users. Either way, the - // memory manager/status reporter is deprecated. -} - -namespace { - std::map<const framework::MemoryAllocationType*, uint32_t> assignColors( - const std::vector<const framework::MemoryAllocationType*>& types) - { - Palette palette(types.size() + 1); - std::map<const framework::MemoryAllocationType*, uint32_t> colors; - uint32_t nextCol = 0; - colors[0] = palette[nextCol++]; - for (std::vector<const framework::MemoryAllocationType*> - ::const_iterator it = types.begin(); it != types.end(); ++it) - { - colors[*it] = palette[nextCol++]; - } - return colors; - } -} - -void -MemoryStatusViewer::reportHtmlStatus(std::ostream& out, - const framework::HttpUrlPath& path) const -{ - vespalib::MonitorGuard monitor(_workerMonitor); - - if (path.getAttribute("page") == "reset") { - } - if (path.getAttribute("interval") == "current") { - Entry& e(*_states[0]); - out << "<pre>" << e._name << ": "; - if (e.containsData()) { - e._data.print(out, true, " "); - } else { - out << "na"; - } - out << "\n</pre>"; - return; - } - const_cast<MemoryStatusViewer*>(this)->grabMemoryUsage(); - framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); - std::vector<const framework::MemoryAllocationType*> allocTypes( - _manager.getAllocationTypes()); - std::map<const framework::MemoryAllocationType*, uint32_t> colors( - assignColors(allocTypes)); - // Print memory usage graph - { - uint32_t mb = 1024 * 1024; - Graph memoryHistory("memhistory", Graph::SCHEME_CUSTOM); - std::vector<Graph::Point> total; - std::vector<Graph::Point> used; - std::vector<Graph::Point> usedWoCache; - uint32_t xval = 0; - for (std::deque<MemoryTimeEntry>::const_iterator it - = _memoryHistory.begin(); it != _memoryHistory.end(); - ++it, ++xval) - { - used.push_back(Graph::Point(xval, it->used)); - usedWoCache.push_back(Graph::Point(xval, it->usedWithoutCache)); - } - used.push_back(Graph::Point( - xval, _states[0]->_data.getUsedSize() / mb)); - usedWoCache.push_back(Graph::Point( - xval, _states[0]->_data.getUsedSizeIgnoringCache() / mb)); - uint32_t totalSize = _states[0]->_maxMemory / mb; - total.push_back(Graph::Point(0, totalSize)); - total.push_back(Graph::Point(xval, totalSize)); - memoryHistory.add(total, "Total memory", Graph::GREEN); - memoryHistory.add(used, "Used memory", Graph::YELLOW); - memoryHistory.add(usedWoCache, "Used memory excluding freeable cache", - Graph::RED); - out << "<p>Memory available for lowest priority (255): " - << _manager.getMemorySizeFreeForPriority(255) << " byte(s).</p>\n"; - out << "<h3>Historic memory usage</h3>\n"; - uint32_t yAxisUnit = ((totalSize / 4) / 256) * 256; - if (yAxisUnit == 0) yAxisUnit = (totalSize / 4); - if (yAxisUnit == 0) yAxisUnit = 1; - uint32_t size = yAxisUnit; - memoryHistory.addYAxisLabel(0, "0 B"); - while (size <= totalSize) { - std::ostringstream label; - if (size % 1024 == 0) { - label << (size / 1024) << " GB"; - } else { - label << size << " MB"; - } - memoryHistory.addYAxisLabel(size, label.str()); - size += yAxisUnit; - } - uint32_t xAxisUnit = ((_memoryHistory.size() / 4) / 24) * 24; - if (xAxisUnit == 0) xAxisUnit = _memoryHistoryPeriod.getTime(); - uint32_t startTime = ((currentTime.getTime() - / _memoryHistoryPeriod.getTime()) - / 24) * 24; - uint32_t stopTime = (currentTime.getTime() - / _memoryHistoryPeriod.getTime()) - - _memoryHistory.size() + 1; - memoryHistory.addXAxisLabel(xval, currentTime.toString()); - bool addedMiddlePoints = false; - while (startTime >= stopTime) { - if (currentTime.getTime() / _memoryHistoryPeriod.getTime() - - startTime > 48) - { - memoryHistory.addXAxisLabel( - (startTime - stopTime), - framework::SecondTime( - startTime * _memoryHistoryPeriod.getTime()) - .toString()); - addedMiddlePoints = true; - } - startTime -= xAxisUnit; - } - if (!addedMiddlePoints && _memoryHistory.size() > 2) { - memoryHistory.addXAxisLabel( - 1, - framework::SecondTime( - stopTime * _memoryHistoryPeriod.getTime()) - .toString()); - } - memoryHistory.setBorders(50, 0, 0, 30); - memoryHistory.setLegendPos(80, 20); - memoryHistory.printCanvas(out, 1000, 250); - memoryHistory.printScript(out, ""); - } - uint32_t maxUsedWithoutCache = 0; - for (uint32_t i=0; i<_states.size(); ++i) { - Entry& e(*_states[i]); - if (!e.containsData() - || e._data.getUsedSizeIgnoringCache() == maxUsedWithoutCache) - { - continue; - } - printSnapshot(out, e, colors); - maxUsedWithoutCache = e._data.getUsedSizeIgnoringCache(); - } - out << "<h3>Raw output of stored data</h3>\n" - << "<pre>\n"; - monitor.unlock(); - printDebugOutput(out); - out << "</pre>\n"; - out << "<h2>Memory used for metrics. (Not tracked in memory manager)</h2>\n" - << "<pre>\n" - << _metricManager.getMemoryConsumption(_metricManager.getMetricLock())->toString() - << "\n</pre>\n"; -} - -void -MemoryStatusViewer::run(framework::ThreadHandle& thread) -{ - while (!thread.interrupted()) { - vespalib::MonitorGuard monitor(_workerMonitor); - framework::SecondTime currentTime( - _component.getClock().getTimeInSeconds()); - if (_lastHistoryUpdate + _memoryHistoryPeriod <= currentTime - || _states[0]->_timeTaken + _memoryHistoryPeriod <= currentTime) - { - grabMemoryUsage(); - _processedTime = currentTime; - LOG(spam, "Done processing time %" PRIu64, currentTime.getTime()); - thread.registerTick(framework::PROCESS_CYCLE); - } else { - monitor.wait(thread.getWaitTime()); - thread.registerTick(framework::WAIT_CYCLE); - } - } -} - -// You should have worker monitor when calling this function -void -MemoryStatusViewer::grabMemoryUsage() -{ - framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); - MemoryState state(_component.getClock(), 0); - _manager.getState(state, true); - - if (_lastHistoryUpdate + _memoryHistoryPeriod <= currentTime) { - LOG(spam, "Adding another %" PRIu64 " sec entry to memory history.", - _memoryHistoryPeriod.getTime()); - // Add history once an hour - uint32_t mb = 1024 * 1024; - _memoryHistory.push_back(MemoryTimeEntry( - state.getMaxSnapshot().getUsedSize() / mb, - state.getMaxSnapshot().getUsedSizeIgnoringCache() / mb)); - if (_memoryHistory.size() > _memoryHistorySize) { - if (_memoryHistoryPeriod != framework::SecondTime(60 * 60)) { - uint32_t periodDiff = 60 * 60 / _memoryHistoryPeriod.getTime(); - std::deque<MemoryTimeEntry> newHistory; - uint32_t count = 0; - MemoryTimeEntry entry(0, 0); - for (std::deque<MemoryTimeEntry>::const_iterator it - = _memoryHistory.begin(); - it != _memoryHistory.end(); ++it) - { - entry.keepMax(*it); - if (++count == periodDiff) { - newHistory.push_back(entry); - entry = MemoryTimeEntry(0, 0); - count = 0; - } - } - if (entry.used != 0) { - newHistory.push_back(entry); - } - _memoryHistory.swap(newHistory); - _memoryHistoryPeriod = framework::SecondTime(60 * 60); - } - } - _lastHistoryUpdate += _memoryHistoryPeriod; - if (_lastHistoryUpdate + _allowedSlackPeriod < currentTime) { - LOGBP(warning, "Memory history is supposed to be tracked every %" - PRIu64 " seconds, but %" PRIu64" seconds have passed " - "since last update. Memory history graph will be " - "incorrect.", - _memoryHistoryPeriod.getTime(), - (currentTime - _lastHistoryUpdate + _memoryHistoryPeriod) - .getTime()); - _lastHistoryUpdate = currentTime; - } - } - LOG(spam, "Overwriting current with snapshot using %" PRIu64 " bytes.", - state.getCurrentSnapshot().getUsedSize()); - _states[0]->assign(state.getCurrentSnapshot(), - state.getTotalSize(), currentTime); - for (uint32_t i=1, n=_states.size(); i<n; ++i) { - if (currentTime - _states[i]->_timeTaken >= _states[i]->_maxAge - || state.getMaxSnapshot().getUsedSize() - > _states[i]->_data.getUsedSize()) - { - LOG(spam, "Updating period %s usage. Old usage was %" PRIu64 ". " - "Last set at %" PRIu64, - _states[i]->_name.c_str(), _states[i]->_data.getUsedSize(), - _states[i]->_timeTaken.getTime()); - _states[i]->assign(state.getMaxSnapshot(), - state.getTotalSize(), currentTime); - } - } -} - -void -MemoryStatusViewer::notifyThread() const -{ - vespalib::MonitorGuard monitor(_workerMonitor); - monitor.broadcast(); -} - -void -MemoryStatusViewer::printDebugOutput(std::ostream& out) const -{ - vespalib::MonitorGuard monitor(_workerMonitor); - for (uint32_t i=0; i<_states.size(); ++i) { - Entry& e(*_states[i]); - out << e._name << ": "; - if (e.containsData()) { - out << e._timeTaken.toString() << " Max memory " << e._maxMemory << " "; - e._data.print(out, true, " "); - } else { - out << "na"; - } - out << "\n\n"; - } -} - -} // storage diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h deleted file mode 100644 index 2c7510d371d..00000000000 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::MemoryStatusViewer - * - * \brief Generates status to access through status pages. - * - * Keeps a history of the largest memory inprints seen historically. This is - * done be defining periods, where a period is always a multiplum of the length - * of the period shorter than it. The last entry will store the biggest memory - * imprint ever seen, and the earlier entries will show biggest for their time - * period. - * - * To avoid having all periods cleared once the biggest period resets, the - * periods keep data for each of the periods one size below it. Thus, a year - * keeps data for 12 months, a month for 30 days, and so on. - * - * The memory state objects are divided in 3 parts. Current memory data, max - * memory data since since reset and counts for how often various events have - * happened. - * - * The counts will have their total count values stored in the current entry. - * When the next period is updated getting a copy of these counts, we can see - * how many counts have happened recently, by taking the current entry and - * subtract those accounted for earlier. - * - * The current memory data will not be interesting for anything than to show the - * actual now values in the current entry. - * - * The max since reset values will be the values used for the various periods. - * When a period is updated with new data for a subpart of their period, the - * max seen data is reset in the period in front, such that a lower maximum - * can be found. - */ - -#pragma once - -#include <vespa/storage/common/storagecomponent.h> -#include <vespa/storageframework/defaultimplementation/memory/memorystate.h> -#include <vespa/storageframework/generic/status/htmlstatusreporter.h> -#include <vespa/vespalib/util/document_runnable.h> -#include <vespa/vespalib/util/sync.h> -#include <deque> -#include <vector> - - -namespace metrics { - class MetricManager; -} - -namespace storage { - -class StorageServerInterface; - -class MemoryStatusViewer : public framework::HtmlStatusReporter, - private framework::Runnable -{ -public: - typedef framework::defaultimplementation::MemoryState::SnapShot SnapShot; - struct Entry { - typedef std::shared_ptr<Entry> SP; - - std::string _name; - framework::SecondTime _maxAge; - framework::SecondTime _timeTaken; - SnapShot _data; - uint64_t _maxMemory; - - Entry(const std::string& name, framework::Clock&, - framework::SecondTime maxAge); - bool containsData() const { return (_maxMemory != 0); } - - void assign(const SnapShot& snapshot, uint64_t maxMemory, - framework::SecondTime time) - { - _data = snapshot; - _maxMemory = maxMemory; - _timeTaken = time; - } - }; - - struct MemoryTimeEntry { - uint64_t used; - uint64_t usedWithoutCache; - - MemoryTimeEntry(uint64_t u, uint64_t wo) - : used(u), usedWithoutCache(wo) {} - - void keepMax(const MemoryTimeEntry& e) { - used = (used > e.used ? used : e.used); - usedWithoutCache = (usedWithoutCache > e.usedWithoutCache - ? usedWithoutCache : e.usedWithoutCache); - } - }; - -private: - framework::Component _component; - framework::defaultimplementation::MemoryManager& _manager; - const metrics::MetricManager& _metricManager; - vespalib::Monitor _workerMonitor; - - std::vector<Entry::SP> _states; - std::deque<MemoryTimeEntry> _memoryHistory; - uint32_t _memoryHistorySize; - framework::SecondTime _memoryHistoryPeriod; - framework::SecondTime _allowedSlackPeriod; - framework::SecondTime _lastHistoryUpdate; - framework::Thread::UP _thread; - framework::SecondTime _processedTime; - - void addEntry(const std::string& name, uint32_t maxAge) { - _states.push_back(Entry::SP(new Entry(name, _component.getClock(), - framework::SecondTime(maxAge)))); - } - void run(framework::ThreadHandle&) override; - void grabMemoryUsage(); - void printSnapshot(std::ostream& out, Entry& entry, - std::map<const framework::MemoryAllocationType*, - uint32_t>& colors) const; - -public: - MemoryStatusViewer( - framework::defaultimplementation::MemoryManager&, - const metrics::MetricManager&, - StorageComponentRegister&); - ~MemoryStatusViewer(); - - void reportHtmlHeaderAdditions(std::ostream&, const framework::HttpUrlPath&) const override; - void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; - - /** Useful for testing. */ - framework::SecondTime getProcessedTime() const { return _processedTime; } - void notifyThread() const; - void printDebugOutput(std::ostream&) const; - -}; - -} diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp index 876067e4e15..720371a68f1 100644 --- a/storage/src/vespa/storage/persistence/messages.cpp +++ b/storage/src/vespa/storage/persistence/messages.cpp @@ -6,17 +6,14 @@ using document::BucketSpace; namespace storage { -GetIterCommand::GetIterCommand(framework::MemoryToken::UP token, - const document::Bucket &bucket, +GetIterCommand::GetIterCommand(const document::Bucket &bucket, const spi::IteratorId iteratorId, uint32_t maxByteSize) : api::InternalCommand(ID), - _token(std::move(token)), _bucket(bucket), _iteratorId(iteratorId), _maxByteSize(maxByteSize) { - assert(_token.get()); } GetIterCommand::~GetIterCommand() { } @@ -38,7 +35,6 @@ GetIterCommand::makeReply() { GetIterReply::GetIterReply(GetIterCommand& cmd) : api::InternalReply(ID, cmd), - _token(cmd.releaseMemoryToken()), _bucket(cmd.getBucket()), _completed(false) { } diff --git a/storage/src/vespa/storage/persistence/messages.h b/storage/src/vespa/storage/persistence/messages.h index 7bbeea8a12a..ba7f5979569 100644 --- a/storage/src/vespa/storage/persistence/messages.h +++ b/storage/src/vespa/storage/persistence/messages.h @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storageframework/generic/memory/memorytoken.h> #include <vespa/storageapi/message/internal.h> #include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/bucket.h> @@ -14,7 +13,6 @@ namespace storage { class GetIterCommand : public api::InternalCommand { private: - mutable framework::MemoryToken::UP _token; document::Bucket _bucket; spi::IteratorId _iteratorId; uint32_t _maxByteSize; @@ -24,8 +22,7 @@ public: typedef std::unique_ptr<GetIterCommand> UP; typedef std::shared_ptr<GetIterCommand> SP; - GetIterCommand(framework::MemoryToken::UP token, - const document::Bucket &bucket, + GetIterCommand(const document::Bucket &bucket, const spi::IteratorId iteratorId, uint32_t maxByteSize); ~GetIterCommand(); @@ -44,13 +41,11 @@ public: void print(std::ostream& out, bool verbose, const std::string& indent) const override; private: - framework::MemoryToken::UP releaseMemoryToken() { return std::move(_token); } friend class GetIterReply; }; class GetIterReply : public api::InternalReply { private: - framework::MemoryToken::UP _token; document::Bucket _bucket; std::vector<spi::DocEntry::UP> _entries; bool _completed; diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 3d0a0fdfa14..c0238922a91 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -11,7 +11,6 @@ vespa_add_library(storage_storageserver documentapiconverter.cpp fnetlistener.cpp mergethrottler.cpp - messageallocationtypes.cpp messagesink.cpp opslogger.cpp priorityconverter.cpp diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index eae51b90165..b214ef4539a 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -96,13 +96,6 @@ StorageTransportContext::StorageTransportContext(std::unique_ptr<RPCRequestWrapp StorageTransportContext::~StorageTransportContext() { } -const framework::MemoryAllocationType& -CommunicationManager::getAllocationType(api::StorageMessage& msg) const -{ - return _messageAllocTypes.getType(msg.getType().getId()); -} - - void CommunicationManager::receiveStorageReply(const std::shared_ptr<api::StorageReply>& reply) { @@ -296,8 +289,7 @@ CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, co _configUri(configUri), _closed(false), _bucketResolver(std::make_unique<PlaceHolderBucketResolver>()), - _docApiConverter(configUri, *_bucketResolver), - _messageAllocTypes(_component.getMemoryManager()) + _docApiConverter(configUri, *_bucketResolver) { _component.registerMetricUpdateHook(*this, framework::SecondTime(5)); _component.registerMetric(_metrics); @@ -483,33 +475,9 @@ CommunicationManager::process(const std::shared_ptr<api::StorageMessage>& msg) void CommunicationManager::enqueue(const std::shared_ptr<api::StorageMessage> & msg) { - using MemoryToken = framework::MemoryToken; assert(msg.get()); - - const uint32_t memoryFootprint = msg->getMemoryFootprint(); - MemoryToken::UP token = _component.getMemoryManager().allocate(getAllocationType(*msg), memoryFootprint * 2, - memoryFootprint * 2, msg->getPriority()); - - if (token) { - msg->setMemoryToken(std::move(token)); - - LOG(spam, "Enq storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); - _eventQueue.enqueue(msg); - } else { - _metrics.failedDueToTooLittleMemory.inc(); - std::ostringstream ost; - ost << "Failed to aquire " << (memoryFootprint * 2) - << " bytes of memory to handle command of type " - << msg->getType() << "\n"; - LOG(spam, "%s", ost.str().c_str()); - api::StorageCommand* cmd(dynamic_cast<api::StorageCommand*>(msg.get())); - - if (cmd) { - std::shared_ptr<api::StorageReply> reply(cmd->makeReply()); - reply->setResult(api::ReturnCode(api::ReturnCode::BUSY, ost.str())); - sendReply(reply); - } - } + LOG(spam, "Enq storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); + _eventQueue.enqueue(msg); } bool diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index 4cf3f33e6ea..89b76583cb5 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -11,7 +11,6 @@ #pragma once #include "communicationmanagermetrics.h" -#include "messageallocationtypes.h" #include "documentapiconverter.h" #include <vespa/storage/common/storagelink.h> #include <vespa/storage/common/storagecomponent.h> @@ -171,10 +170,7 @@ private: std::unique_ptr<BucketResolver> _bucketResolver; DocumentApiConverter _docApiConverter; framework::Thread::UP _thread; - MessageAllocationTypes _messageAllocTypes; - const framework::MemoryAllocationType& - getAllocationType(api::StorageMessage& msg) const; void updateMetrics(const MetricLockGuard &) override; // Test needs access to configure() for live reconfig testing. diff --git a/storage/src/vespa/storage/storageserver/framework.h b/storage/src/vespa/storage/storageserver/framework.h index 724b4fe0944..f0ea1d71aa6 100644 --- a/storage/src/vespa/storage/storageserver/framework.h +++ b/storage/src/vespa/storage/storageserver/framework.h @@ -18,7 +18,6 @@ #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> -#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> namespace storage { @@ -27,7 +26,6 @@ struct Framework { // Typedefs to simplify the remainder of the interface typedef StorageComponentRegisterImpl CompReg; typedef framework::defaultimplementation::RealClock RealClock; - typedef framework::defaultimplementation::MemoryManager MemoryManager; /** * You can provide your own clock implementation. Useful in testing where @@ -48,20 +46,10 @@ struct Framework { */ FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); } - /** - * Get the memory manager. Components that wants to print status of memory - * manager need access to the actual implementation. - */ - MemoryManager& getMemoryManager() { return _memoryManager; } - - void setMaximumMemoryUsage(uint64_t max); - private: CompReg _componentRegister; framework::Clock::UP _clock; framework::defaultimplementation::ThreadPoolImpl _threadPool; - framework::defaultimplementation::AllocationLogic* _memoryLogic; - MemoryManager _memoryManager; }; diff --git a/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp b/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp deleted file mode 100644 index ac2d99a439c..00000000000 --- a/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "messageallocationtypes.h" -#include <vespa/storageapi/messageapi/storagemessage.h> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/stllike/asciistream.h> - -namespace storage { - -MessageAllocationTypes::MessageAllocationTypes(framework::MemoryManagerInterface& manager) -{ - using api::MessageType; - using framework::MemoryAllocationType; - - _types.resize(MessageType::MESSAGETYPE_MAX_ID); - _types[MessageType::DOCBLOCK_ID] = &manager.registerAllocationType(MemoryAllocationType("MESSAGE_DOCBLOCK")); - _types[MessageType::DOCBLOCK_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MESSAGE_DOCBLOCK_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GET", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::GET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::INTERNAL_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::INTERNAL")); - _types[MessageType::INTERNAL_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::INTERNAL_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::PUT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::PUT", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::PUT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::PUT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::REMOVE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVE", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::REMOVE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::REVERT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REVERT")); - _types[MessageType::REVERT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REVERT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::VISITOR_CREATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_CREATE", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::VISITOR_CREATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_CREATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::VISITOR_DESTROY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_DESTROY")); - _types[MessageType::VISITOR_DESTROY_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_DESTROY_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::REQUESTBUCKETINFO_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REQUESTBUCKETINFO")); - _types[MessageType::REQUESTBUCKETINFO_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REQUESTBUCKETINFO_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::NOTIFYBUCKETCHANGE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::NOTIFYBUCKETCHANGE")); - _types[MessageType::NOTIFYBUCKETCHANGE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::NOTIFYBUCKETCHANGE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::CREATEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::CREATEBUCKET")); - _types[MessageType::CREATEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::CREATEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::MERGEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MERGEBUCKET")); - _types[MessageType::MERGEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MERGEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::DELETEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DELETEBUCKET")); - _types[MessageType::DELETEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DELETEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SETNODESTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETNODESTATE", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SETNODESTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETNODESTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETNODESTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETNODESTATE", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETNODESTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETNODESTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SETSYSTEMSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETSYSTEMSTATE", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SETSYSTEMSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETSYSTEMSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETSYSTEMSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETSYSTEMSTATE", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETSYSTEMSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETSYSTEMSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETBUCKETDIFF_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETDIFF", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETBUCKETDIFF_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETDIFF_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::APPLYBUCKETDIFF_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::APPLYBUCKETDIFF", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::APPLYBUCKETDIFF_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::APPLYBUCKETDIFF_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::VISITOR_INFO_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_INFO")); - _types[MessageType::VISITOR_INFO_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_INFO_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SEARCHRESULT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SEARCHRESULT")); - _types[MessageType::SEARCHRESULT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SEARCHRESULT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SPLITBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SPLITBUCKET")); - _types[MessageType::SPLITBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SPLITBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::JOINBUCKETS_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::JOINBUCKETS")); - _types[MessageType::JOINBUCKETS_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::JOINBUCKETS_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::SETBUCKETSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETBUCKETSTATE")); - _types[MessageType::SETBUCKETSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETBUCKETSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::MULTIOPERATION_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MULTIOPERATION", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::MULTIOPERATION_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MULTIOPERATION_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::DOCUMENTSUMMARY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTSUMMARY")); - _types[MessageType::DOCUMENTSUMMARY_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTSUMMARY_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::MAPVISITOR_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MAPVISITOR")); - _types[MessageType::MAPVISITOR_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MAPVISITOR_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::STATBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::STATBUCKET", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::STATBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::STATBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::GETBUCKETLIST_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETLIST", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::GETBUCKETLIST_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETLIST_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::DOCUMENTLIST_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTLIST")); - _types[MessageType::DOCUMENTLIST_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTLIST_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::UPDATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::UPDATE", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::UPDATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::UPDATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::EMPTYBUCKETS_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::EMPTYBUCKETS")); - _types[MessageType::EMPTYBUCKETS_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::EMPTYBUCKETS_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::REMOVELOCATION_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVELOCATION", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::REMOVELOCATION_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVELOCATION_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::QUERYRESULT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::QUERYRESULT")); - _types[MessageType::QUERYRESULT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::QUERYRESULT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE)); - _types[MessageType::BATCHPUTREMOVE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHPUTREMOVE", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::BATCHPUTREMOVE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHPUTREMOVE_REPLY", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::BATCHDOCUMENTUPDATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHDOCUMENTUPDATE", framework::MemoryAllocationType::EXTERNAL_LOAD)); - _types[MessageType::BATCHDOCUMENTUPDATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHDOCUMENTUPDATE_REPLY", framework::MemoryAllocationType::EXTERNAL_LOAD)); -} - -const framework::MemoryAllocationType& -MessageAllocationTypes::getType(uint32_t type) const { - if (_types.size() > size_t(type) && _types[type] != 0) { - return *_types[type]; - } - vespalib::asciistream ost; - ost << "No type registered with value " << type << "."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); -} - -} // storage diff --git a/storage/src/vespa/storage/storageserver/messageallocationtypes.h b/storage/src/vespa/storage/storageserver/messageallocationtypes.h deleted file mode 100644 index 8f7e8e4b3c3..00000000000 --- a/storage/src/vespa/storage/storageserver/messageallocationtypes.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::MessageAllocationTypes - * - * \brief Memory allocation types for messages in storage. - */ -#pragma once - -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> -#include <vector> - -namespace storage { - -class MessageAllocationTypes { - std::vector<const framework::MemoryAllocationType*> _types; - -public: - MessageAllocationTypes(framework::MemoryManagerInterface& manager); - - const framework::MemoryAllocationType& getType(uint32_t type) const; -}; - -} - diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 855efaed6aa..ba1556bd3b9 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -7,7 +7,6 @@ #include "storagemetricsset.h" #include "storagenodecontext.h" -#include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h> #include <vespa/storage/frameworkimpl/status/statuswebserver.h> #include <vespa/storage/frameworkimpl/thread/deadlockdetector.h> #include <vespa/storage/common/statusmetricconsumer.h> @@ -117,7 +116,6 @@ StorageNode::initialize() // and store them away, while having the config lock. subscribeToConfigs(); - _context.getMemoryManager().setMaximumMemoryUsage(_serverConfig->memorytouse); updateUpgradeFlag(*_clusterConfig); // First update some basics that doesn't depend on anything else to be @@ -157,11 +155,6 @@ StorageNode::initialize() initializeNodeSpecific(); - _memoryStatusViewer.reset(new MemoryStatusViewer( - _context.getMemoryManager(), - _context.getComponentRegister().getMetricManager(), - _context.getComponentRegister())); - _statusMetrics.reset(new StatusMetricConsumer( _context.getComponentRegister(), _context.getComponentRegister().getMetricManager())); _stateReporter.reset(new StateReporter( @@ -259,14 +252,6 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) DIFFERWARN(clusterName, "Cannot alter cluster name of node live"); DIFFERWARN(nodeIndex, "Cannot alter node index of node live"); DIFFERWARN(isDistributor, "Cannot alter role of node live"); - { - if (DIFFER(memorytouse)) { - LOG(info, "Live config update: Memory to use changed from %" PRId64 " to %" PRId64 ".", - oldC.memorytouse, newC.memorytouse); - ASSIGN(memorytouse); - _context.getMemoryManager().setMaximumMemoryUsage(newC.memorytouse); - } - } _serverConfig.reset(new StorServerConfig(oldC)); _newServerConfig.reset(); (void)updated; @@ -413,10 +398,6 @@ StorageNode::shutdown() LOG(debug, "Deleting state reporter"); _stateReporter.reset(); } - if (_memoryStatusViewer) { - LOG(debug, "Deleting memory status viewer"); - _memoryStatusViewer.reset(); - } if (_stateManager) { LOG(debug, "Deleting state manager"); _stateManager.reset(); diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index 9b727ef3e0c..e9d3004be68 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -121,8 +121,6 @@ private: // Depends on bucket databases and stop() functionality std::unique_ptr<DeadLockDetector> _deadLockDetector; - // Depends on dead lock detector and threadpool - std::unique_ptr<MemoryStatusViewer> _memoryStatusViewer; // Depends on metric manager std::unique_ptr<StatusMetricConsumer> _statusMetrics; // Depends on metric manager diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp index 2e1aa52e68d..75e1f12773f 100644 --- a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp +++ b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp @@ -2,31 +2,15 @@ #include "storagenodecontext.h" -#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h> - namespace storage { -using framework::defaultimplementation::AllocationLogic; -using framework::defaultimplementation::PriorityMemoryLogic; - StorageNodeContext::StorageNodeContext(ComponentRegister::UP compReg, framework::Clock::UP clock) : _componentRegister(std::move(compReg)), _clock(std::move(clock)), - _threadPool(*_clock), - _memoryLogic(new PriorityMemoryLogic(*_clock, 1024 * 1024 * 1024)), - _memoryManager(AllocationLogic::UP(_memoryLogic)) + _threadPool(*_clock) { _componentRegister->setClock(*_clock); _componentRegister->setThreadPool(_threadPool); - _componentRegister->setMemoryManager(_memoryManager); -} - -void -StorageNodeContext::setMaximumMemoryUsage(uint64_t max) -{ - using storage::framework::defaultimplementation::PriorityMemoryLogic; - dynamic_cast<PriorityMemoryLogic*>(_memoryLogic) - ->setMaximumMemoryUsage(max); } } // storage diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.h b/storage/src/vespa/storage/storageserver/storagenodecontext.h index 0149f975f63..eabca618bfb 100644 --- a/storage/src/vespa/storage/storageserver/storagenodecontext.h +++ b/storage/src/vespa/storage/storageserver/storagenodecontext.h @@ -18,7 +18,6 @@ #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> -#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> namespace storage { @@ -27,7 +26,6 @@ struct StorageNodeContext { // Typedefs to simplify the remainder of the interface typedef StorageComponentRegisterImpl ComponentRegister; typedef framework::defaultimplementation::RealClock RealClock; - typedef framework::defaultimplementation::MemoryManager MemoryManager; /** * Get the actual component register. Available as the actual type as the @@ -42,14 +40,6 @@ struct StorageNodeContext { */ FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); } - /** - * Get the memory manager. Components that wants to print status of memory - * manager need access to the actual implementation. - */ - MemoryManager& getMemoryManager() { return _memoryManager; } - - void setMaximumMemoryUsage(uint64_t max); - protected: // Initialization has been split in two as subclass needs to initialize // component register before sending it on. @@ -59,8 +49,6 @@ private: ComponentRegister::UP _componentRegister; framework::Clock::UP _clock; framework::defaultimplementation::ThreadPoolImpl _threadPool; - framework::defaultimplementation::AllocationLogic* _memoryLogic; - MemoryManager _memoryManager; }; diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index b91e38e035e..0d230745875 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -2,7 +2,6 @@ #include "visitor.h" #include "visitormetrics.h" -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storage/persistence/messages.h> @@ -267,8 +266,7 @@ Visitor::Visitor(StorageComponent& component) _id(), _controlDestination(), _dataDestination(), - _documentSelection(), - _memoryManager(0) + _documentSelection() { } @@ -603,10 +601,6 @@ Visitor::start(api::VisitorId id, api::StorageMessage::Id cmdId, _documentPriority = documentPriority; _state = STATE_RUNNING; - if (_memoryAllocType == 0) { - _memoryAllocType = &_component.getMemoryManager() - .getAllocationType("VISITOR_BUFFER"); - } LOG(debug, "Starting visitor '%s' for %" PRIu64 " buckets from %" PRIu64 " to " "%" PRIu64 ". First is %s. Max pending replies: %u, include " @@ -803,17 +797,7 @@ Visitor::onCreateIteratorReply( LOG(debug, "Visitor '%s' starting to visit bucket %s.", _id.c_str(), bucketId.toString().c_str()); - framework::MemoryToken::UP token( - _memoryManager->allocate( - *_memoryAllocType, _docBlockSize, _docBlockSize, _priority)); - if (token.get() == 0) { - // Not enough memory - return; - } - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), bucket, - bucketState.getIteratorId(), - _docBlockSize)); + auto cmd = std::make_shared<GetIterCommand>(bucket, bucketState.getIteratorId(), _docBlockSize); cmd->setLoadType(_initiatingCmd->getLoadType()); cmd->getTrace().setLevel(_traceLevel); cmd->setPriority(_priority); @@ -1223,19 +1207,8 @@ Visitor::getIterators() it = _bucketStates.erase(it); continue; } - framework::MemoryToken::UP token( - _memoryManager->allocate( - *_memoryAllocType, _docBlockSize, _docBlockSize, - _priority)); - if (token.get() == 0) { - // Not enough memory - return true; - } - std::shared_ptr<GetIterCommand> cmd( - new GetIterCommand(std::move(token), - bucketState.getBucket(), - bucketState.getIteratorId(), - _docBlockSize)); + auto cmd = std::make_shared<GetIterCommand>( + bucketState.getBucket(), bucketState.getIteratorId(), _docBlockSize); cmd->setLoadType(_initiatingCmd->getLoadType()); cmd->getTrace().setLevel(_traceLevel); cmd->setPriority(_priority); diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 4436312032f..f84b105f1ed 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -327,7 +327,6 @@ private: protected: // These variables should not be altered after visitor starts. This not // controlled by locks. - const framework::MemoryAllocationType* _memoryAllocType; VisitorMessageHandler* _messageHandler; VisitorMessageSession::UP _messageSession; documentapi::Priority::Value _documentPriority; @@ -339,7 +338,6 @@ protected: std::string _documentSelectionString; std::unique_ptr<document::OrderingSpecification> _ordering; vdslib::VisitorStatistics _visitorStatistics; - framework::MemoryManagerInterface* _memoryManager; bool isCompletedCalled() const { return _calledCompletedVisitor; } @@ -365,8 +363,6 @@ public: const api::StorageMessageAddress* getDataDestination() const { return _dataDestination.get(); } // Can't be null if attached - void setAllocationType(const framework::MemoryAllocationType& mat) - { _memoryAllocType = &mat; } void setMaxPending(unsigned int maxPending) { _visitorOptions._maxPending = maxPending; } @@ -384,8 +380,6 @@ public: { _docBlockTimeout = timeout; } void setVisitorInfoTimeout(framework::MilliSecTime timeout) { _visitorInfoTimeout = timeout; } - void setMemoryManager(framework::MemoryManagerInterface& mm) - { _memoryManager = &mm; } void setOwnNodeIndex(uint16_t nodeIndex) { _ownNodeIndex = nodeIndex; } void setBucketSpace(document::BucketSpace bucketSpace) { _bucketSpace = bucketSpace; } diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp index deb38bb927d..f207c4436b1 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.cpp +++ b/storage/src/vespa/storage/visiting/visitormanager.cpp @@ -7,7 +7,6 @@ #include "countvisitor.h" #include "testvisitor.h" #include "recoveryvisitor.h" -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storage/common/statusmessages.h> #include <vespa/config/common/exceptions.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> @@ -46,10 +45,6 @@ VisitorManager::VisitorManager(const config::ConfigUri & configUri, _enforceQueueUse(false), _visitorFactories(externalFactories) { - _component.getMemoryManager().registerAllocationType( - framework::MemoryAllocationType( - "VISITOR_BUFFER", - framework::MemoryAllocationType::EXTERNAL_LOAD)); _configFetcher.subscribe<vespa::config::content::core::StorVisitorConfig>(configUri.getConfigId(), this); _configFetcher.start(); _component.registerMetric(*_metrics); diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index d3fed86b741..4743ac66054 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -2,7 +2,6 @@ #include "visitorthread.h" #include "messages.h" -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/select/bodyfielddetector.h> #include <vespa/document/select/orderingselector.h> @@ -100,9 +99,7 @@ VisitorThread::VisitorThread(uint32_t threadIndex, _timeBetweenTicks(1000), _component(componentRegister, getThreadName(threadIndex)), _messageSessionFactory(messageSessionFac), - _visitorFactories(visitorFactories), - _memoryBufferAlloc( - _component.getMemoryManager().getAllocationType("VISITOR_BUFFER")) + _visitorFactories(visitorFactories) { framework::MilliSecTime maxProcessingTime(30 * 1000); framework::MilliSecTime waitTime(1000); @@ -473,8 +470,6 @@ VisitorThread::onCreateVisitor( cmd->getInstanceId().c_str(), errors.str().c_str()); break; } - visitor->setAllocationType(_memoryBufferAlloc); - visitor->setMemoryManager(_component.getMemoryManager()); // Set visitor parameters if (cmd->getMaximumPendingReplyCount() != 0) { visitor->setMaxPending(cmd->getMaximumPendingReplyCount()); diff --git a/storage/src/vespa/storage/visiting/visitorthread.h b/storage/src/vespa/storage/visiting/visitorthread.h index 1f1459ccb2b..97ca70f3761 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.h +++ b/storage/src/vespa/storage/visiting/visitorthread.h @@ -90,7 +90,6 @@ class VisitorThread : public framework::Runnable, framework::Thread::UP _thread; VisitorMessageSessionFactory& _messageSessionFactory; VisitorFactory::Map& _visitorFactories; - const framework::MemoryAllocationType& _memoryBufferAlloc; public: VisitorThread(uint32_t threadIndex, |