summaryrefslogtreecommitdiffstats
path: root/storage
diff options
context:
space:
mode:
authorTor Brede Vekterli <vekterli@yahoo-inc.com>2017-12-04 11:31:29 +0000
committerTor Brede Vekterli <vekterli@yahoo-inc.com>2017-12-04 13:06:46 +0000
commit952a877611d657cfa0166b14699c8731b18f7587 (patch)
treec1b4330b9dc512c3ffbf845e626857ddcd801982 /storage
parentd164fbb93e277ef23ab610320a7cf8556e3c036e (diff)
Remove memory manager component from content layer
We already have resource utilization tracking in both MessageBus and the search core. The memory manager has never been auto-scaled based on the hardware present and adds a _lot_ of complexity without having any known instances where it has actually saved the day. Removing it also removes a mutex on the message hot path. If we need such functionality in the future, should design a lock-free solution. Cleanup
Diffstat (limited to 'storage')
-rw-r--r--storage/CMakeLists.txt2
-rw-r--r--storage/src/tests/CMakeLists.txt1
-rw-r--r--storage/src/tests/common/teststorageapp.h1
-rw-r--r--storage/src/tests/frameworkimpl/memory/CMakeLists.txt8
-rw-r--r--storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp170
-rw-r--r--storage/src/tests/persistence/filestorage/filestormanagertest.cpp57
-rw-r--r--storage/src/tests/persistence/persistencetestutils.cpp1
-rw-r--r--storage/src/tests/persistence/persistencetestutils.h1
-rw-r--r--storage/src/tests/storageserver/communicationmanagertest.cpp1
-rw-r--r--storage/src/vespa/storage/CMakeLists.txt1
-rw-r--r--storage/src/vespa/storage/bucketdb/bucketmanager.cpp9
-rw-r--r--storage/src/vespa/storage/bucketdb/bucketmanager.h2
-rw-r--r--storage/src/vespa/storage/common/statusmetricconsumer.cpp15
-rw-r--r--storage/src/vespa/storage/common/statusmetricconsumer.h1
-rw-r--r--storage/src/vespa/storage/distributor/bucketdbupdater.h1
-rw-r--r--storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h2
-rw-r--r--storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt8
-rw-r--r--storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h20
-rw-r--r--storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp661
-rw-r--r--storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h137
-rw-r--r--storage/src/vespa/storage/persistence/messages.cpp6
-rw-r--r--storage/src/vespa/storage/persistence/messages.h7
-rw-r--r--storage/src/vespa/storage/storageserver/CMakeLists.txt1
-rw-r--r--storage/src/vespa/storage/storageserver/communicationmanager.cpp38
-rw-r--r--storage/src/vespa/storage/storageserver/communicationmanager.h4
-rw-r--r--storage/src/vespa/storage/storageserver/framework.h12
-rw-r--r--storage/src/vespa/storage/storageserver/messageallocationtypes.cpp100
-rw-r--r--storage/src/vespa/storage/storageserver/messageallocationtypes.h24
-rw-r--r--storage/src/vespa/storage/storageserver/storagenode.cpp19
-rw-r--r--storage/src/vespa/storage/storageserver/storagenode.h2
-rw-r--r--storage/src/vespa/storage/storageserver/storagenodecontext.cpp18
-rw-r--r--storage/src/vespa/storage/storageserver/storagenodecontext.h12
-rw-r--r--storage/src/vespa/storage/visiting/visitor.cpp35
-rw-r--r--storage/src/vespa/storage/visiting/visitor.h6
-rw-r--r--storage/src/vespa/storage/visiting/visitormanager.cpp5
-rw-r--r--storage/src/vespa/storage/visiting/visitorthread.cpp7
-rw-r--r--storage/src/vespa/storage/visiting/visitorthread.h1
37 files changed, 21 insertions, 1375 deletions
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 24d5a2d27f2..5c24d4adb3b 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -35,7 +35,6 @@ vespa_define_module(
src/vespa/storage/distributor/operations/external
src/vespa/storage/distributor/operations/idealstate
src/vespa/storage/frameworkimpl/component
- src/vespa/storage/frameworkimpl/memory
src/vespa/storage/frameworkimpl/status
src/vespa/storage/frameworkimpl/thread
src/vespa/storage/persistence
@@ -60,7 +59,6 @@ vespa_define_module(
src/tests/common
src/tests/common/hostreporter
src/tests/distributor
- src/tests/frameworkimpl/memory
src/tests/frameworkimpl/status
src/tests/persistence
src/tests/persistence/common
diff --git a/storage/src/tests/CMakeLists.txt b/storage/src/tests/CMakeLists.txt
index 7747e8bacc1..9962949edde 100644
--- a/storage/src/tests/CMakeLists.txt
+++ b/storage/src/tests/CMakeLists.txt
@@ -13,7 +13,6 @@ vespa_add_executable(storage_testrunner_app TEST
storage_testdistributor
storage_testpersistence
storage_testfilestorage
- storage_testmemory
storage_teststatus
)
diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h
index 6906d41ac47..46744ecb3c6 100644
--- a/storage/src/tests/common/teststorageapp.h
+++ b/storage/src/tests/common/teststorageapp.h
@@ -24,7 +24,6 @@
#include <vespa/storage/storageserver/framework.h>
#include <vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h>
#include <vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
#include <vespa/storageframework/defaultimplementation/component/testcomponentregister.h>
#include <vespa/persistence/spi/persistenceprovider.h>
diff --git a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt b/storage/src/tests/frameworkimpl/memory/CMakeLists.txt
deleted file mode 100644
index e5a364e02a3..00000000000
--- a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(storage_testmemory TEST
- SOURCES
- memorystatusviewertest.cpp
- DEPENDS
- storage
- storage_testcommon
-)
diff --git a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp b/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp
deleted file mode 100644
index 1ad4b16751d..00000000000
--- a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp
+++ /dev/null
@@ -1,170 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/metrics/metrics.h>
-#include <vespa/metrics/metricmanager.h>
-#include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h>
-#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h>
-#include <tests/common/teststorageapp.h>
-#include <vespa/vdstestlib/cppunit/macros.h>
-#include <vespa/vespalib/util/exceptions.h>
-#include <boost/lexical_cast.hpp>
-
-namespace storage {
-
-struct MemoryStatusViewerTest : public CppUnit::TestFixture
-{
- static const int maxMemory = 1000;
- std::unique_ptr<TestServiceLayerApp> _node;
- std::unique_ptr<framework::defaultimplementation::MemoryManager> _memMan;
-
- void setUp() override;
-
- void testEmptyState();
- void testSnapshots();
-
- CPPUNIT_TEST_SUITE(MemoryStatusViewerTest);
- CPPUNIT_TEST(testEmptyState);
- CPPUNIT_TEST(testSnapshots);
- CPPUNIT_TEST_SUITE_END();
-};
-
-CPPUNIT_TEST_SUITE_REGISTRATION(MemoryStatusViewerTest);
-
-void
-MemoryStatusViewerTest::setUp()
-{
- _node.reset(new TestServiceLayerApp(DiskCount(2)));
- framework::defaultimplementation::PriorityMemoryLogic* logic(
- new framework::defaultimplementation::PriorityMemoryLogic(
- _node->getClock(), maxMemory));
- logic->setMinJumpToUpdateMax(1);
- _memMan.reset(new framework::defaultimplementation::MemoryManager(
- framework::defaultimplementation::AllocationLogic::UP(logic)));
-}
-
-void
-MemoryStatusViewerTest::testEmptyState()
-{
- // Add a memory manager, and add a bit of load to it, so it's not
- // totally empty.
- StorageComponent component(_node->getComponentRegister(), "test");
-
- metrics::MetricManager mm;
- MemoryStatusViewer viewer(
- *_memMan, mm, _node->getComponentRegister());
- std::ostringstream actual;
- viewer.reportStatus(actual, framework::HttpUrlPath("/"));
- CPPUNIT_ASSERT_MATCH_REGEX(".*Plotr.LineChart.*", actual.str());
- CPPUNIT_ASSERT_MATCH_REGEX(
- ".*Current: 1970-01-01 00:00:00 Max memory 1000 SnapShot\\(Used 0, w/o cache 0\\).*",
- actual.str());
- CPPUNIT_ASSERT_MATCH_REGEX(
- ".*Last hour: na.*", actual.str());
-}
-
-namespace {
- void waitForProcessedTime(
- const MemoryStatusViewer& viewer, framework::SecondTime time,
- framework::SecondTime timeout = framework::SecondTime(30))
- {
- framework::defaultimplementation::RealClock clock;
- framework::MilliSecTime endTime(
- clock.getTimeInMillis() + timeout.getMillis());
- framework::SecondTime processedTime(0);
- while (clock.getTimeInMillis() < endTime) {
- processedTime = viewer.getProcessedTime();
- if (processedTime >= time) return;
- FastOS_Thread::Sleep(1);
- }
- std::ostringstream ost;
- ost << "Timed out waiting " << timeout << " ms for time " << time
- << " to be processed. Currently time is only processed up to "
- << processedTime;
- throw new vespalib::IllegalStateException(ost.str(), VESPA_STRLOC);
- }
-}
-
-#define ASSERT_MEMORY(output, period, maxmem, used, usedwocache) \
-{ \
- std::string::size_type _pos1_(output.find(period)); \
- std::string::size_type _pos2_(output.find("Max memory", _pos1_)); \
- std::string::size_type _pos3_(output.find("SnapShot", _pos2_)); \
- std::string _maxMemory_(output.substr(_pos2_ + 11, _pos3_ - _pos2_ - 12)); \
- std::string::size_type _pos4_(output.find(",", _pos3_)); \
- std::string _used_(output.substr(_pos3_ + 14, _pos4_ - _pos3_ - 14)); \
- std::string::size_type _pos5_(output.find(")", _pos4_)); \
- std::string _usedwo_(output.substr(_pos4_ + 12, _pos5_ - _pos4_ - 12)); \
- std::ostringstream _failure_; \
- _failure_ << "Wrong match in period " << period << " in output:\n" \
- << output << "\nFor value: "; \
- \
- CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Max memory", \
- uint64_t(maxmem), boost::lexical_cast<uint64_t>(_maxMemory_)); \
- CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Used memory", \
- uint64_t(used), boost::lexical_cast<uint64_t>(_used_)); \
- CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Used memory w/o cache", \
- uint64_t(usedwocache), boost::lexical_cast<uint64_t>(_usedwo_)); \
-}
-
-void
-MemoryStatusViewerTest::testSnapshots()
-{
- // Add a memory manager, and add a bit of load to it, so it's not
- // totally empty.
- StorageComponent component(_node->getComponentRegister(), "test");
- const framework::MemoryAllocationType putAlloc(
- component.getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType("PUT")));
- const framework::MemoryAllocationType getAlloc(
- component.getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType("GET")));
-
- framework::MemoryToken::UP put = _memMan->allocate(putAlloc, 0, 100, 80);
- framework::MemoryToken::UP get = _memMan->allocate(getAlloc, 30, 200, 50);
- framework::MemoryToken::UP get2 = _memMan->allocate(getAlloc, 70, 150, 60);
-
- metrics::MetricManager mm;
- MemoryStatusViewer viewer(*_memMan, mm, _node->getComponentRegister());
-
- _node->getClock().addSecondsToTime(1000);
- viewer.notifyThread();
- waitForProcessedTime(viewer, framework::SecondTime(1000));
-
- std::ostringstream actual;
- viewer.printDebugOutput(actual);
- //std::cerr << actual.str() << "\n";
- ASSERT_MEMORY(actual.str(), "Current", 1000, 450, 450);
- ASSERT_MEMORY(actual.str(), "Last hour", 1000, 450, 450);
- ASSERT_MEMORY(actual.str(), "Last ever", 1000, 450, 450);
-
- put = _memMan->allocate(putAlloc, 0, 50, 80);
- get = _memMan->allocate(getAlloc, 100, 140, 50);
- get2 = _memMan->allocate(getAlloc, 20, 100, 70);
-
- _node->getClock().addSecondsToTime(3600);
- viewer.notifyThread();
- waitForProcessedTime(viewer, framework::SecondTime(4600));
-
- actual.str("");
- viewer.printDebugOutput(actual);
- //std::cerr << actual.str() << "\n";
- ASSERT_MEMORY(actual.str(), "Current", 1000, 290, 290);
- ASSERT_MEMORY(actual.str(), "Last hour", 1000, 540, 540);
- ASSERT_MEMORY(actual.str(), "Last ever", 1000, 540, 540);
-
- get.reset();
-
- _node->getClock().addSecondsToTime(3600);
- viewer.notifyThread();
- waitForProcessedTime(viewer, framework::SecondTime(4600 + 3600));
-
- actual.str("");
- viewer.printDebugOutput(actual);
- //std::cerr << actual.str() << "\n";
- ASSERT_MEMORY(actual.str(), "Current", 1000, 150, 150);
- ASSERT_MEMORY(actual.str(), "Last hour", 1000, 290, 290);
- ASSERT_MEMORY(actual.str(), "Last ever", 1000, 540, 540);
-
-}
-
-} // storage
diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
index 67b93108139..af0082fa788 100644
--- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
+++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
@@ -231,8 +231,6 @@ struct FileStorManagerTest : public CppUnit::TestFixture {
fprintf(stderr, "%s\n", e.what());
}
_testdoctype1 = _node->getTypeRepo()->getDocumentType("testdoctype1");
- _node->getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType("VISITOR_BUFFER"));
}
void putDoc(DummyStorageLink& top,
@@ -2010,19 +2008,11 @@ FileStorManagerTest::testVisiting()
top.reset();
// Visit bucket with no split, using no selection
{
- framework::MemoryToken::UP token(
- _node->getMemoryManager().allocate(
- _node->getMemoryManager().getAllocationType(
- "VISITOR_BUFFER"),
- 16*1024,
- 16*1024,
- 127));
spi::IteratorId iterId(createIterator(top, ids[0], "true"));
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), makeDocumentBucket(ids[0]), iterId, 16*1024));
+ auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[0]), iterId, 16*1024);
top.sendDown(cmd);
top.waitForMessages(1, _waitTime);
- CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies());
+ CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies());
std::shared_ptr<GetIterReply> reply(
std::dynamic_pointer_cast<GetIterReply>(top.getReply(0)));
CPPUNIT_ASSERT(reply.get());
@@ -2039,15 +2029,7 @@ FileStorManagerTest::testVisiting()
ids[1],
"testdoctype1.hstringval = \"John Doe\""));
while (true) {
- framework::MemoryToken::UP token(
- _node->getMemoryManager().allocate(
- _node->getMemoryManager().getAllocationType(
- "VISITOR_BUFFER"),
- 16*1024,
- 16*1024,
- 127));
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), makeDocumentBucket(ids[1]), iterId, 16*1024));
+ auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[1]), iterId, 16*1024);
top.sendDown(cmd);
top.waitForMessages(1, _waitTime);
CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies());
@@ -2078,18 +2060,10 @@ FileStorManagerTest::testVisiting()
true));
uint32_t totalDocs = 0;
while (true) {
- framework::MemoryToken::UP token(
- _node->getMemoryManager().allocate(
- _node->getMemoryManager().getAllocationType(
- "VISITOR_BUFFER"),
- 16*1024,
- 16*1024,
- 127));
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), makeDocumentBucket(ids[1]), iterId, 16*1024));
+ auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[1]), iterId, 16*1024);
top.sendDown(cmd);
top.waitForMessages(1, _waitTime);
- CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies());
+ CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies());
std::shared_ptr<GetIterReply> reply(
std::dynamic_pointer_cast<GetIterReply>(
top.getReply(0)));
@@ -2799,15 +2773,8 @@ FileStorManagerTest::testGetIter()
// Sending a getiter request that will only visit some of the docs
spi::IteratorId iterId(createIterator(top, bid, ""));
{
- framework::MemoryToken::UP token(
- _node->getMemoryManager().allocate(
- _node->getMemoryManager().getAllocationType(
- "VISITOR_BUFFER"),
- 2048,
- 2048,
- 127));
std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), makeDocumentBucket(bid), iterId, 2048));
+ new GetIterCommand(makeDocumentBucket(bid), iterId, 2048));
top.sendDown(cmd);
top.waitForMessages(1, _waitTime);
CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies());
@@ -2837,18 +2804,10 @@ FileStorManagerTest::testGetIter()
CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult());
}
{
- framework::MemoryToken::UP token(
- _node->getMemoryManager().allocate(
- _node->getMemoryManager().getAllocationType(
- "VISITOR_BUFFER"),
- 2048,
- 2048,
- 127));
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), makeDocumentBucket(bid), iterId, 2048));
+ auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(bid), iterId, 2048);
top.sendDown(cmd);
top.waitForMessages(1, _waitTime);
- CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies());
+ CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies());
std::shared_ptr<GetIterReply> reply(
std::dynamic_pointer_cast<GetIterReply>(
top.getReply(0)));
diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp
index fcfbcd0a78b..9d3a3d5f008 100644
--- a/storage/src/tests/persistence/persistencetestutils.cpp
+++ b/storage/src/tests/persistence/persistencetestutils.cpp
@@ -11,7 +11,6 @@
#include <vespa/vespalib/util/exceptions.h>
using document::DocumentType;
-using storage::framework::defaultimplementation::AllocationLogic;
using storage::spi::test::makeSpiBucket;
using document::test::makeDocumentBucket;
diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h
index ee87925a0bc..36b1fef7285 100644
--- a/storage/src/tests/persistence/persistencetestutils.h
+++ b/storage/src/tests/persistence/persistencetestutils.h
@@ -8,7 +8,6 @@
#include <vespa/storage/persistence/persistenceutil.h>
#include <vespa/storage/common/messagesender.h>
#include <vespa/storage/common/storagecomponent.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
#include <vespa/persistence/spi/persistenceprovider.h>
#include <vespa/persistence/dummyimpl/dummypersistence.h>
#include <vespa/document/base/testdocman.h>
diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp
index 5111368dd1c..cd5dd2a01a4 100644
--- a/storage/src/tests/storageserver/communicationmanagertest.cpp
+++ b/storage/src/tests/storageserver/communicationmanagertest.cpp
@@ -6,7 +6,6 @@
#include <vespa/messagebus/rpcmessagebus.h>
#include <vespa/storageapi/message/persistence.h>
#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h>
-#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h>
#include <tests/common/teststorageapp.h>
#include <tests/common/dummystoragelink.h>
#include <tests/common/testhelper.h>
diff --git a/storage/src/vespa/storage/CMakeLists.txt b/storage/src/vespa/storage/CMakeLists.txt
index 69846cafde2..d854c603097 100644
--- a/storage/src/vespa/storage/CMakeLists.txt
+++ b/storage/src/vespa/storage/CMakeLists.txt
@@ -12,7 +12,6 @@ vespa_add_library(storage
$<TARGET_OBJECTS:storage_bucketmover>
$<TARGET_OBJECTS:storage_thread>
$<TARGET_OBJECTS:storage_status>
- $<TARGET_OBJECTS:storage_memory>
$<TARGET_OBJECTS:storage_component>
INSTALL lib64
DEPENDS
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
index 677ca0a6309..aa9158f9e5b 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
@@ -34,7 +34,6 @@ BucketManager::BucketManager(const config::ConfigUri & configUri,
: StorageLinkQueued("Bucket manager", compReg),
framework::StatusReporter("bucketdb", "Bucket database"),
_configUri(configUri),
- _bucketDBMemoryToken(),
_workerLock(),
_workerCond(),
_clusterStateLock(),
@@ -48,12 +47,6 @@ BucketManager::BucketManager(const config::ConfigUri & configUri,
_requestsCurrentlyProcessing(0),
_component(compReg, "bucketmanager")
{
- const framework::MemoryAllocationType& allocType(
- _component.getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType("DATABASE")));
- _bucketDBMemoryToken = _component.getMemoryManager().allocate(
- allocType, 0, 0, api::StorageMessage::HIGH);
- assert(_bucketDBMemoryToken.get() != 0);
_metrics->setDisks(_component.getDiskCount());
_component.registerStatusPage(*this);
_component.registerMetric(*_metrics);
@@ -222,8 +215,6 @@ BucketManager::updateMetrics(bool updateDocCount)
LOG(debug, "Iterating bucket database to update metrics%s%s",
updateDocCount ? "" : ", minusedbits only",
_doneInitialized ? "" : ", server is not done initializing");
- uint64_t dbMemSize = _component.getBucketSpaceRepo().getBucketMemoryUsage();
- _bucketDBMemoryToken->resize(dbMemSize, dbMemSize);
uint32_t diskCount = _component.getDiskCount();
if (!updateDocCount || _doneInitialized) {
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h
index 5e4e928dfc7..8413c5a0f42 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.h
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h
@@ -20,7 +20,6 @@
#include <vespa/storage/common/servicelayercomponent.h>
#include <vespa/storage/common/storagelinkqueued.h>
#include <vespa/storageapi/message/bucket.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/generic/metric/metricupdatehook.h>
#include <vespa/storageframework/generic/status/statusreporter.h>
@@ -47,7 +46,6 @@ private:
config::ConfigUri _configUri;
uint32_t _chunkLevel;
- framework::MemoryToken::UP _bucketDBMemoryToken;
BucketInfoRequestMap _bucketInfoRequests;
/**
diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp
index de44116b316..d458a821e02 100644
--- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp
+++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "statusmetricconsumer.h"
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/generic/status/htmlstatusreporter.h>
#include <boost/assign.hpp>
#include <boost/lexical_cast.hpp>
@@ -26,17 +25,8 @@ StatusMetricConsumer::StatusMetricConsumer(
_component(compReg, "statusmetricsconsumer"),
_name(name),
_startTime(_component.getClock().getTimeInSeconds()),
- _processedTime(0),
- _metricMemoryToken()
+ _processedTime(0)
{
- const framework::MemoryAllocationType& allocType(
- _component.getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType(
- "METRICS", framework::MemoryAllocationType::FORCE_ALLOCATE)
- ));
- _metricMemoryToken = _component.getMemoryManager().allocate(
- allocType, 0, 0, api::StorageMessage::HIGH);
- assert(_metricMemoryToken.get() != 0);
LOG(debug, "Started metrics consumer");
setlocale(LC_NUMERIC, "");
_component.registerMetricUpdateHook(*this, framework::SecondTime(3600));
@@ -51,8 +41,7 @@ void
StatusMetricConsumer::updateMetrics(const MetricLockGuard & guard)
{
metrics::MemoryConsumption::UP mc(_manager.getMemoryConsumption(guard));
- uint32_t usage = mc->getTotalMemoryUsage();
- _metricMemoryToken->resize(usage, usage);
+ // TODO is this hook needed anymore?
}
vespalib::string
diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.h b/storage/src/vespa/storage/common/statusmetricconsumer.h
index e3bae51cf99..6f93f51cfdf 100644
--- a/storage/src/vespa/storage/common/statusmetricconsumer.h
+++ b/storage/src/vespa/storage/common/statusmetricconsumer.h
@@ -53,7 +53,6 @@ private:
vespalib::Monitor _waiter;
framework::SecondTime _startTime;
framework::SecondTime _processedTime;
- std::unique_ptr<framework::MemoryToken> _metricMemoryToken;
void writeXmlTags(std::ostream& out,
const vespalib::StringTokenizer& name,
diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h
index a3c9804c2b4..29e8d3f6221 100644
--- a/storage/src/vespa/storage/distributor/bucketdbupdater.h
+++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h
@@ -13,7 +13,6 @@
#include <vespa/vdslib/state/clusterstate.h>
#include <vespa/storage/common/storagelink.h>
#include <vespa/storageframework/generic/clock/timer.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageapi/messageapi/messagehandler.h>
#include <set>
#include <deque>
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h
index e8480902549..8d31a228543 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h
@@ -6,7 +6,6 @@
#include <vespa/storageapi/messageapi/storagereply.h>
#include <vespa/storageapi/messageapi/maintenancecommand.h>
#include <vespa/document/bucket/bucketid.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
namespace storage::distributor {
@@ -229,7 +228,6 @@ protected:
bool _ok;
api::StorageMessage::Priority _priority;
- framework::MemoryToken::UP _memoryToken;
/**
* Checks if the given bucket is blocked by any pending messages to any
diff --git a/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt b/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt
deleted file mode 100644
index 2a91ead2b60..00000000000
--- a/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(storage_memory OBJECT
- SOURCES
- memorystatusviewer.cpp
- DEPENDS
- AFTER
- storage_storageconfig
-)
diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h b/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h
deleted file mode 100644
index fd47bec1424..00000000000
--- a/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::MemorySnapshotList
- *
- * \brief Holds a historic list of MemoryStates.
- *
- */
-
-#pragma once
-
-#include <storage/frameworkimpl/memory/memorystate.h>
-
-namespace storage {
-
-class MemorySnapshotList : public vespalib::Printable {
- std::map<uint64_t time, MemoryState> _snapshots;
-};
-
-} // storage
-
diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp
deleted file mode 100644
index dd07a61f27b..00000000000
--- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp
+++ /dev/null
@@ -1,661 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memorystatusviewer.h"
-#include <vespa/storage/storageutil/graph.h>
-#include <vespa/storage/storageutil/palette.h>
-#include <vespa/storage/storageutil/piechart.h>
-#include <vespa/metrics/metricmanager.h>
-#include <vespa/storageapi/messageapi/storagemessage.h>
-#include <algorithm>
-
-#include <vespa/log/bufferedlogger.h>
-LOG_SETUP(".memory.status.viewer");
-
-using storage::framework::defaultimplementation::MemoryState;
-
-namespace storage {
-
-MemoryStatusViewer::Entry::Entry(
- const std::string& name, framework::Clock& clock,
- framework::SecondTime maxAge)
- : _name(name),
- _maxAge(maxAge),
- _timeTaken(clock.getTimeInSeconds()),
- _data(),
- _maxMemory(0)
-{
-}
-
-MemoryStatusViewer::MemoryStatusViewer(
- framework::defaultimplementation::MemoryManager& mm,
- const metrics::MetricManager& metricMan,
- StorageComponentRegister& compReg)
- : framework::HtmlStatusReporter("memorymanager", "Memory Manager"),
- _component(compReg, "memorystatusviewer"),
- _manager(mm),
- _metricManager(metricMan),
- _workerMonitor(),
- _states(),
- _memoryHistory(),
- _memoryHistorySize(24 * 31),
- _memoryHistoryPeriod(60),
- _allowedSlackPeriod(6),
- _lastHistoryUpdate(_component.getClock().getTimeInSeconds()),
- _processedTime(0)
-{
- addEntry("Current", 0);
- addEntry("Last hour", 60 * 60);
- addEntry("Last day", 24 * 60 * 60);
- addEntry("Last month", 4 * 7 * 24 * 60 * 60);
- addEntry("Last ever", std::numeric_limits<uint32_t>::max());
-
- framework::MilliSecTime maxProcessingTime(60 * 1000);
- _thread = _component.startThread(*this, maxProcessingTime,
- framework::MilliSecTime(1000));
- _component.registerStatusPage(*this);
-}
-
-MemoryStatusViewer::~MemoryStatusViewer()
-{
- if (_thread.get() != 0) {
- _thread->interrupt();
- {
- vespalib::MonitorGuard monitor(_workerMonitor);
- monitor.signal();
- }
- _thread->join();
- }
-}
-
-namespace {
- struct Group {
- std::set<const framework::MemoryAllocationType*> types;
- api::StorageMessage::Priority minPri;
- api::StorageMessage::Priority maxPri;
- MemoryState::Entry entry;
-
- Group(const framework::MemoryAllocationType& type,
- api::StorageMessage::Priority pri,
- const MemoryState::Entry& e)
- : types(),
- minPri(pri),
- maxPri(pri),
- entry(e)
- {
- types.insert(&type);
- }
- };
-
- struct GroupSizeOrder {
- bool operator()(const Group& g1, const Group& g2) {
- return (g1.entry._currentUsedSize > g2.entry._currentUsedSize);
- }
- };
- struct GroupAllocsOrder {
- bool operator()(const Group& g1, const Group& g2) {
- return (g1.entry._totalUserCount > g2.entry._totalUserCount);
- }
- };
- struct GroupMinAllocsOrder {
- bool operator()(const Group& g1, const Group& g2) {
- return (g1.entry._minimumCount > g2.entry._minimumCount);
- }
- };
- struct GroupDeniedAllocsOrder {
- bool operator()(const Group& g1, const Group& g2) {
- return (g1.entry._deniedCount > g2.entry._deniedCount);
- }
- };
-
- std::vector<Group> collapsePriorities(MemoryStatusViewer::Entry& entry) {
- std::vector<Group> groups;
- const MemoryState::SnapShot& ss(entry._data);
- for (MemoryState::AllocationMap::const_iterator it
- = ss.getAllocations().begin();
- it != ss.getAllocations().end(); ++it)
- {
- std::unique_ptr<Group> group;
- for (MemoryState::PriorityMap::const_iterator it2
- = it->second.begin(); it2 != it->second.end(); ++it2)
- {
- if (group.get() == 0) {
- group.reset(new Group(
- *it->first, it2->first, it2->second));
- } else {
- group->entry += it2->second;
- group->minPri = std::min(group->minPri, it2->first);
- group->maxPri = std::max(group->maxPri, it2->first);
- }
- }
- if (group.get() != 0) {
- groups.push_back(*group);
- }
- }
- return groups;
- }
-
- std::vector<Group> groupLoad(uint32_t groupCount, uint64_t minSize,
- uint32_t minEntries,
- MemoryStatusViewer::Entry& entry)
- {
- assert(groupCount > 1);
- std::vector<Group> groups(collapsePriorities(entry));
- if (groups.size() == 0) return groups;
- std::sort(groups.begin(), groups.end(), GroupSizeOrder());
- assert(groups.front().entry._currentUsedSize
- >= groups.back().entry._currentUsedSize);
- while (groups.size() > minEntries
- && (groups.size() > groupCount
- || groups[groups.size() - 2].entry._currentUsedSize
- < minSize))
- {
- Group& nextButLast(groups[groups.size() - 2]);
- Group& last(groups.back());
- if (last.entry._currentUsedSize > 0) {
- nextButLast.entry += last.entry;
- nextButLast.minPri = std::min(nextButLast.minPri, last.minPri);
- nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri);
- nextButLast.types.insert(*last.types.begin());
- }
- groups.pop_back();
- }
- return groups;
- }
-
- std::vector<Group> groupAllocs(uint32_t groupCount, uint64_t minSize,
- uint32_t minEntries,
- MemoryStatusViewer::Entry& entry)
- {
- assert(groupCount > 1);
- std::vector<Group> groups(collapsePriorities(entry));
- if (groups.size() == 0) return groups;
- std::sort(groups.begin(), groups.end(), GroupAllocsOrder());
- assert(groups.front().entry._totalUserCount
- >= groups.back().entry._totalUserCount);
- while (groups.size() > minEntries
- && (groups.size() > groupCount
- || groups[groups.size() - 2].entry._totalUserCount
- < minSize))
- {
- Group& nextButLast(groups[groups.size() - 2]);
- Group& last(groups.back());
- nextButLast.entry += last.entry;
- nextButLast.minPri = std::min(nextButLast.minPri, last.minPri);
- nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri);
- nextButLast.types.insert(*last.types.begin());
- groups.pop_back();
- }
- return groups;
- }
-
- std::vector<Group> groupMinAllocs(uint32_t groupCount, uint64_t minSize,
- uint32_t minEntries,
- MemoryStatusViewer::Entry& entry)
- {
- assert(groupCount > 1);
- std::vector<Group> groups(collapsePriorities(entry));
- if (groups.size() == 0) return groups;
- std::sort(groups.begin(), groups.end(), GroupMinAllocsOrder());
- assert(groups.front().entry._minimumCount
- >= groups.back().entry._minimumCount);
- while (groups.size() > minEntries
- && (groups.size() > groupCount
- || groups[groups.size() - 2].entry._minimumCount
- < minSize))
- {
- Group& nextButLast(groups[groups.size() - 2]);
- Group& last(groups.back());
- nextButLast.entry += last.entry;
- nextButLast.minPri = std::min(nextButLast.minPri, last.minPri);
- nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri);
- nextButLast.types.insert(*last.types.begin());
- groups.pop_back();
- }
- return groups;
- }
-
- std::vector<Group> groupDeniedAllocs(uint32_t groupCount, uint64_t minSize,
- uint32_t minEntries,
- MemoryStatusViewer::Entry& entry)
- {
- assert(groupCount > 1);
- std::vector<Group> groups(collapsePriorities(entry));
- if (groups.size() == 0) return groups;
- std::sort(groups.begin(), groups.end(), GroupDeniedAllocsOrder());
- assert(groups.front().entry._deniedCount
- >= groups.back().entry._deniedCount);
- while (groups.size() > minEntries
- && (groups.size() > groupCount
- || groups[groups.size() - 2].entry._deniedCount
- < minSize))
- {
- Group& nextButLast(groups[groups.size() - 2]);
- Group& last(groups.back());
- nextButLast.entry += last.entry;
- nextButLast.minPri = std::min(nextButLast.minPri, last.minPri);
- nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri);
- nextButLast.types.insert(*last.types.begin());
- groups.pop_back();
- }
- return groups;
- }
-}
-
-void
-MemoryStatusViewer::printSnapshot(
- std::ostream& out, Entry& entry,
- std::map<const framework::MemoryAllocationType*,
- uint32_t>& colors) const
-{
- out << "<h4>" << entry._name << " - Taken at "
- << entry._timeTaken.toString() << "</h4>\n"
- << "<table><tr><td>\n"
- << "<b>Memory usage";
- if (entry._name != "Current") {
- out << ", maxed at " << framework::SecondTime(entry._timeTaken);
- }
- out << " with "
- << (entry._data.getUsedSizeIgnoringCache() / (1024 * 1024))
- << " MB.</b><br>\n";
- std::string piename = entry._name;
- std::replace(piename.begin(), piename.end(), ' ', '_');
- uint64_t freeSize = entry._maxMemory - entry._data.getUsedSize();
- // Memory usage pie
- uint64_t minSize = freeSize / 20;
- std::vector<Group> groups(groupLoad(20, minSize, 5, entry));
- PieChart chart(piename, PieChart::SCHEME_CUSTOM);
- chart.printLabels(false);
- for (uint32_t i=0; i<groups.size(); ++i) {
- std::string name = "Other";
- if (groups[i].types.size() == 1) {
- name = (*groups[i].types.begin())->getName();
- }
- uint32_t mbytes = groups[i].entry._currentUsedSize / (1024 * 1024);
- std::ostringstream ost;
- ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri);
- if (groups[i].minPri != groups[i].maxPri) {
- ost << " - " << static_cast<uint16_t>(groups[i].maxPri);
- }
- ost << " (" << mbytes << " MB)";
- name = ost.str();
- if (groups[i].entry._currentUsedSize > 0) {
- chart.add(groups[i].entry._currentUsedSize, name,
- colors[*groups[i].types.begin()]);
- }
- }
- {
- std::ostringstream ost;
- ost << "Free (" << (freeSize / (1024 * 1024)) << " MB)";
- chart.add(freeSize, ost.str(), colors[0]);
- }
- chart.printCanvas(out, 750, 300);
- out << "\n\n";
- chart.printScript(out, "");
- out << "\n\n";
- // Total allocations pie
- out << "</td><td>\n";
- PieChart allocChart(piename + "Alloc", PieChart::SCHEME_CUSTOM);
- allocChart.printLabels(false);
- groups = groupAllocs(20, 100, 5, entry);
- uint64_t totalAllocs = 0;
- for (uint32_t i=0; i<groups.size(); ++i) {
- std::string name = "Other";
- if (groups[i].types.size() == 1) {
- name = (*groups[i].types.begin())->getName();
- }
- uint32_t allocs = groups[i].entry._totalUserCount;
- totalAllocs += allocs;
- std::ostringstream ost;
- ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri);
- if (groups[i].minPri != groups[i].maxPri) {
- ost << " - " << static_cast<uint16_t>(groups[i].maxPri);
- }
- ost << " (" << allocs << " allocations)";
- name = ost.str();
- if (groups[i].entry._totalUserCount > 0) {
- allocChart.add(groups[i].entry._totalUserCount, name,
- colors[*groups[i].types.begin()]);
- }
- }
- out << "<b>Allocations, totalling " << totalAllocs << "</b><br>\n";
- allocChart.printCanvas(out, 750, 300);
- out << "\n\n";
- allocChart.printScript(out, "");
- out << "\n\n";
- out << "</td></tr><tr><td>\n";
- PieChart minChart(piename + "Min", PieChart::SCHEME_CUSTOM);
- minChart.printLabels(false);
- groups = groupMinAllocs(20, 100, 5, entry);
- uint64_t totalMinAllocs = 0;
- for (uint32_t i=0; i<groups.size(); ++i) {
- std::string name = "Other";
- if (groups[i].types.size() == 1) {
- name = (*groups[i].types.begin())->getName();
- }
- uint32_t allocs = groups[i].entry._minimumCount;
- totalMinAllocs += allocs;
- std::ostringstream ost;
- ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri);
- if (groups[i].minPri != groups[i].maxPri) {
- ost << " - " << static_cast<uint16_t>(groups[i].maxPri);
- }
- ost << " (" << allocs << " min allocations)";
- name = ost.str();
- if (groups[i].entry._minimumCount > 0) {
- minChart.add(groups[i].entry._minimumCount, name,
- colors[*groups[i].types.begin()]);
- }
- }
- out << "<b>Minimum allocations, totalling " << totalMinAllocs
- << "</b><br>\n";
- if (totalMinAllocs > 0) {
- minChart.printCanvas(out, 750, 300);
- out << "\n\n";
- minChart.printScript(out, "");
- out << "\n\n";
- }
- out << "</td><td>\n";
- PieChart deniedChart(piename + "Denied", PieChart::SCHEME_CUSTOM);
- deniedChart.printLabels(false);
- groups = groupDeniedAllocs(20, 100, 5, entry);
- uint64_t totalDeniedAllocs = 0;
- for (uint32_t i=0; i<groups.size(); ++i) {
- std::string name = "Other";
- if (groups[i].types.size() == 1) {
- name = (*groups[i].types.begin())->getName();
- }
- uint32_t allocs = groups[i].entry._deniedCount;
- totalDeniedAllocs += allocs;
- std::ostringstream ost;
- ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri);
- if (groups[i].minPri != groups[i].maxPri) {
- ost << " - " << static_cast<uint16_t>(groups[i].maxPri);
- }
- ost << " (" << allocs << " denied allocations)";
- name = ost.str();
- if (groups[i].entry._deniedCount > 0) {
- deniedChart.add(groups[i].entry._deniedCount, name,
- colors[*groups[i].types.begin()]);
- }
- }
- out << "<b>Denied allocations, totalling " << totalDeniedAllocs
- << "</b><br>\n";
- if (totalDeniedAllocs > 0) {
- deniedChart.printCanvas(out, 750, 300);
- out << "\n\n";
- deniedChart.printScript(out, "");
- out << "\n\n";
- }
- out << "</td></tr></table>\n";
-}
-
-void
-MemoryStatusViewer::reportHtmlHeaderAdditions(
- std::ostream& out, const framework::HttpUrlPath&) const
-{
- (void) out;
- // FIXME this function used to emit Yahoo-internal links to graph plotting
- // JS files. Obviously, this won't work for external users. Either way, the
- // memory manager/status reporter is deprecated.
-}
-
-namespace {
- std::map<const framework::MemoryAllocationType*, uint32_t> assignColors(
- const std::vector<const framework::MemoryAllocationType*>& types)
- {
- Palette palette(types.size() + 1);
- std::map<const framework::MemoryAllocationType*, uint32_t> colors;
- uint32_t nextCol = 0;
- colors[0] = palette[nextCol++];
- for (std::vector<const framework::MemoryAllocationType*>
- ::const_iterator it = types.begin(); it != types.end(); ++it)
- {
- colors[*it] = palette[nextCol++];
- }
- return colors;
- }
-}
-
-void
-MemoryStatusViewer::reportHtmlStatus(std::ostream& out,
- const framework::HttpUrlPath& path) const
-{
- vespalib::MonitorGuard monitor(_workerMonitor);
-
- if (path.getAttribute("page") == "reset") {
- }
- if (path.getAttribute("interval") == "current") {
- Entry& e(*_states[0]);
- out << "<pre>" << e._name << ": ";
- if (e.containsData()) {
- e._data.print(out, true, " ");
- } else {
- out << "na";
- }
- out << "\n</pre>";
- return;
- }
- const_cast<MemoryStatusViewer*>(this)->grabMemoryUsage();
- framework::SecondTime currentTime(_component.getClock().getTimeInSeconds());
- std::vector<const framework::MemoryAllocationType*> allocTypes(
- _manager.getAllocationTypes());
- std::map<const framework::MemoryAllocationType*, uint32_t> colors(
- assignColors(allocTypes));
- // Print memory usage graph
- {
- uint32_t mb = 1024 * 1024;
- Graph memoryHistory("memhistory", Graph::SCHEME_CUSTOM);
- std::vector<Graph::Point> total;
- std::vector<Graph::Point> used;
- std::vector<Graph::Point> usedWoCache;
- uint32_t xval = 0;
- for (std::deque<MemoryTimeEntry>::const_iterator it
- = _memoryHistory.begin(); it != _memoryHistory.end();
- ++it, ++xval)
- {
- used.push_back(Graph::Point(xval, it->used));
- usedWoCache.push_back(Graph::Point(xval, it->usedWithoutCache));
- }
- used.push_back(Graph::Point(
- xval, _states[0]->_data.getUsedSize() / mb));
- usedWoCache.push_back(Graph::Point(
- xval, _states[0]->_data.getUsedSizeIgnoringCache() / mb));
- uint32_t totalSize = _states[0]->_maxMemory / mb;
- total.push_back(Graph::Point(0, totalSize));
- total.push_back(Graph::Point(xval, totalSize));
- memoryHistory.add(total, "Total memory", Graph::GREEN);
- memoryHistory.add(used, "Used memory", Graph::YELLOW);
- memoryHistory.add(usedWoCache, "Used memory excluding freeable cache",
- Graph::RED);
- out << "<p>Memory available for lowest priority (255): "
- << _manager.getMemorySizeFreeForPriority(255) << " byte(s).</p>\n";
- out << "<h3>Historic memory usage</h3>\n";
- uint32_t yAxisUnit = ((totalSize / 4) / 256) * 256;
- if (yAxisUnit == 0) yAxisUnit = (totalSize / 4);
- if (yAxisUnit == 0) yAxisUnit = 1;
- uint32_t size = yAxisUnit;
- memoryHistory.addYAxisLabel(0, "0 B");
- while (size <= totalSize) {
- std::ostringstream label;
- if (size % 1024 == 0) {
- label << (size / 1024) << " GB";
- } else {
- label << size << " MB";
- }
- memoryHistory.addYAxisLabel(size, label.str());
- size += yAxisUnit;
- }
- uint32_t xAxisUnit = ((_memoryHistory.size() / 4) / 24) * 24;
- if (xAxisUnit == 0) xAxisUnit = _memoryHistoryPeriod.getTime();
- uint32_t startTime = ((currentTime.getTime()
- / _memoryHistoryPeriod.getTime())
- / 24) * 24;
- uint32_t stopTime = (currentTime.getTime()
- / _memoryHistoryPeriod.getTime())
- - _memoryHistory.size() + 1;
- memoryHistory.addXAxisLabel(xval, currentTime.toString());
- bool addedMiddlePoints = false;
- while (startTime >= stopTime) {
- if (currentTime.getTime() / _memoryHistoryPeriod.getTime()
- - startTime > 48)
- {
- memoryHistory.addXAxisLabel(
- (startTime - stopTime),
- framework::SecondTime(
- startTime * _memoryHistoryPeriod.getTime())
- .toString());
- addedMiddlePoints = true;
- }
- startTime -= xAxisUnit;
- }
- if (!addedMiddlePoints && _memoryHistory.size() > 2) {
- memoryHistory.addXAxisLabel(
- 1,
- framework::SecondTime(
- stopTime * _memoryHistoryPeriod.getTime())
- .toString());
- }
- memoryHistory.setBorders(50, 0, 0, 30);
- memoryHistory.setLegendPos(80, 20);
- memoryHistory.printCanvas(out, 1000, 250);
- memoryHistory.printScript(out, "");
- }
- uint32_t maxUsedWithoutCache = 0;
- for (uint32_t i=0; i<_states.size(); ++i) {
- Entry& e(*_states[i]);
- if (!e.containsData()
- || e._data.getUsedSizeIgnoringCache() == maxUsedWithoutCache)
- {
- continue;
- }
- printSnapshot(out, e, colors);
- maxUsedWithoutCache = e._data.getUsedSizeIgnoringCache();
- }
- out << "<h3>Raw output of stored data</h3>\n"
- << "<pre>\n";
- monitor.unlock();
- printDebugOutput(out);
- out << "</pre>\n";
- out << "<h2>Memory used for metrics. (Not tracked in memory manager)</h2>\n"
- << "<pre>\n"
- << _metricManager.getMemoryConsumption(_metricManager.getMetricLock())->toString()
- << "\n</pre>\n";
-}
-
-void
-MemoryStatusViewer::run(framework::ThreadHandle& thread)
-{
- while (!thread.interrupted()) {
- vespalib::MonitorGuard monitor(_workerMonitor);
- framework::SecondTime currentTime(
- _component.getClock().getTimeInSeconds());
- if (_lastHistoryUpdate + _memoryHistoryPeriod <= currentTime
- || _states[0]->_timeTaken + _memoryHistoryPeriod <= currentTime)
- {
- grabMemoryUsage();
- _processedTime = currentTime;
- LOG(spam, "Done processing time %" PRIu64, currentTime.getTime());
- thread.registerTick(framework::PROCESS_CYCLE);
- } else {
- monitor.wait(thread.getWaitTime());
- thread.registerTick(framework::WAIT_CYCLE);
- }
- }
-}
-
-// You should have worker monitor when calling this function
-void
-MemoryStatusViewer::grabMemoryUsage()
-{
- framework::SecondTime currentTime(_component.getClock().getTimeInSeconds());
- MemoryState state(_component.getClock(), 0);
- _manager.getState(state, true);
-
- if (_lastHistoryUpdate + _memoryHistoryPeriod <= currentTime) {
- LOG(spam, "Adding another %" PRIu64 " sec entry to memory history.",
- _memoryHistoryPeriod.getTime());
- // Add history once an hour
- uint32_t mb = 1024 * 1024;
- _memoryHistory.push_back(MemoryTimeEntry(
- state.getMaxSnapshot().getUsedSize() / mb,
- state.getMaxSnapshot().getUsedSizeIgnoringCache() / mb));
- if (_memoryHistory.size() > _memoryHistorySize) {
- if (_memoryHistoryPeriod != framework::SecondTime(60 * 60)) {
- uint32_t periodDiff = 60 * 60 / _memoryHistoryPeriod.getTime();
- std::deque<MemoryTimeEntry> newHistory;
- uint32_t count = 0;
- MemoryTimeEntry entry(0, 0);
- for (std::deque<MemoryTimeEntry>::const_iterator it
- = _memoryHistory.begin();
- it != _memoryHistory.end(); ++it)
- {
- entry.keepMax(*it);
- if (++count == periodDiff) {
- newHistory.push_back(entry);
- entry = MemoryTimeEntry(0, 0);
- count = 0;
- }
- }
- if (entry.used != 0) {
- newHistory.push_back(entry);
- }
- _memoryHistory.swap(newHistory);
- _memoryHistoryPeriod = framework::SecondTime(60 * 60);
- }
- }
- _lastHistoryUpdate += _memoryHistoryPeriod;
- if (_lastHistoryUpdate + _allowedSlackPeriod < currentTime) {
- LOGBP(warning, "Memory history is supposed to be tracked every %"
- PRIu64 " seconds, but %" PRIu64" seconds have passed "
- "since last update. Memory history graph will be "
- "incorrect.",
- _memoryHistoryPeriod.getTime(),
- (currentTime - _lastHistoryUpdate + _memoryHistoryPeriod)
- .getTime());
- _lastHistoryUpdate = currentTime;
- }
- }
- LOG(spam, "Overwriting current with snapshot using %" PRIu64 " bytes.",
- state.getCurrentSnapshot().getUsedSize());
- _states[0]->assign(state.getCurrentSnapshot(),
- state.getTotalSize(), currentTime);
- for (uint32_t i=1, n=_states.size(); i<n; ++i) {
- if (currentTime - _states[i]->_timeTaken >= _states[i]->_maxAge
- || state.getMaxSnapshot().getUsedSize()
- > _states[i]->_data.getUsedSize())
- {
- LOG(spam, "Updating period %s usage. Old usage was %" PRIu64 ". "
- "Last set at %" PRIu64,
- _states[i]->_name.c_str(), _states[i]->_data.getUsedSize(),
- _states[i]->_timeTaken.getTime());
- _states[i]->assign(state.getMaxSnapshot(),
- state.getTotalSize(), currentTime);
- }
- }
-}
-
-void
-MemoryStatusViewer::notifyThread() const
-{
- vespalib::MonitorGuard monitor(_workerMonitor);
- monitor.broadcast();
-}
-
-void
-MemoryStatusViewer::printDebugOutput(std::ostream& out) const
-{
- vespalib::MonitorGuard monitor(_workerMonitor);
- for (uint32_t i=0; i<_states.size(); ++i) {
- Entry& e(*_states[i]);
- out << e._name << ": ";
- if (e.containsData()) {
- out << e._timeTaken.toString() << " Max memory " << e._maxMemory << " ";
- e._data.print(out, true, " ");
- } else {
- out << "na";
- }
- out << "\n\n";
- }
-}
-
-} // storage
diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h
deleted file mode 100644
index 2c7510d371d..00000000000
--- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h
+++ /dev/null
@@ -1,137 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::MemoryStatusViewer
- *
- * \brief Generates status to access through status pages.
- *
- * Keeps a history of the largest memory inprints seen historically. This is
- * done be defining periods, where a period is always a multiplum of the length
- * of the period shorter than it. The last entry will store the biggest memory
- * imprint ever seen, and the earlier entries will show biggest for their time
- * period.
- *
- * To avoid having all periods cleared once the biggest period resets, the
- * periods keep data for each of the periods one size below it. Thus, a year
- * keeps data for 12 months, a month for 30 days, and so on.
- *
- * The memory state objects are divided in 3 parts. Current memory data, max
- * memory data since since reset and counts for how often various events have
- * happened.
- *
- * The counts will have their total count values stored in the current entry.
- * When the next period is updated getting a copy of these counts, we can see
- * how many counts have happened recently, by taking the current entry and
- * subtract those accounted for earlier.
- *
- * The current memory data will not be interesting for anything than to show the
- * actual now values in the current entry.
- *
- * The max since reset values will be the values used for the various periods.
- * When a period is updated with new data for a subpart of their period, the
- * max seen data is reset in the period in front, such that a lower maximum
- * can be found.
- */
-
-#pragma once
-
-#include <vespa/storage/common/storagecomponent.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorystate.h>
-#include <vespa/storageframework/generic/status/htmlstatusreporter.h>
-#include <vespa/vespalib/util/document_runnable.h>
-#include <vespa/vespalib/util/sync.h>
-#include <deque>
-#include <vector>
-
-
-namespace metrics {
- class MetricManager;
-}
-
-namespace storage {
-
-class StorageServerInterface;
-
-class MemoryStatusViewer : public framework::HtmlStatusReporter,
- private framework::Runnable
-{
-public:
- typedef framework::defaultimplementation::MemoryState::SnapShot SnapShot;
- struct Entry {
- typedef std::shared_ptr<Entry> SP;
-
- std::string _name;
- framework::SecondTime _maxAge;
- framework::SecondTime _timeTaken;
- SnapShot _data;
- uint64_t _maxMemory;
-
- Entry(const std::string& name, framework::Clock&,
- framework::SecondTime maxAge);
- bool containsData() const { return (_maxMemory != 0); }
-
- void assign(const SnapShot& snapshot, uint64_t maxMemory,
- framework::SecondTime time)
- {
- _data = snapshot;
- _maxMemory = maxMemory;
- _timeTaken = time;
- }
- };
-
- struct MemoryTimeEntry {
- uint64_t used;
- uint64_t usedWithoutCache;
-
- MemoryTimeEntry(uint64_t u, uint64_t wo)
- : used(u), usedWithoutCache(wo) {}
-
- void keepMax(const MemoryTimeEntry& e) {
- used = (used > e.used ? used : e.used);
- usedWithoutCache = (usedWithoutCache > e.usedWithoutCache
- ? usedWithoutCache : e.usedWithoutCache);
- }
- };
-
-private:
- framework::Component _component;
- framework::defaultimplementation::MemoryManager& _manager;
- const metrics::MetricManager& _metricManager;
- vespalib::Monitor _workerMonitor;
-
- std::vector<Entry::SP> _states;
- std::deque<MemoryTimeEntry> _memoryHistory;
- uint32_t _memoryHistorySize;
- framework::SecondTime _memoryHistoryPeriod;
- framework::SecondTime _allowedSlackPeriod;
- framework::SecondTime _lastHistoryUpdate;
- framework::Thread::UP _thread;
- framework::SecondTime _processedTime;
-
- void addEntry(const std::string& name, uint32_t maxAge) {
- _states.push_back(Entry::SP(new Entry(name, _component.getClock(),
- framework::SecondTime(maxAge))));
- }
- void run(framework::ThreadHandle&) override;
- void grabMemoryUsage();
- void printSnapshot(std::ostream& out, Entry& entry,
- std::map<const framework::MemoryAllocationType*,
- uint32_t>& colors) const;
-
-public:
- MemoryStatusViewer(
- framework::defaultimplementation::MemoryManager&,
- const metrics::MetricManager&,
- StorageComponentRegister&);
- ~MemoryStatusViewer();
-
- void reportHtmlHeaderAdditions(std::ostream&, const framework::HttpUrlPath&) const override;
- void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override;
-
- /** Useful for testing. */
- framework::SecondTime getProcessedTime() const { return _processedTime; }
- void notifyThread() const;
- void printDebugOutput(std::ostream&) const;
-
-};
-
-}
diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp
index 876067e4e15..720371a68f1 100644
--- a/storage/src/vespa/storage/persistence/messages.cpp
+++ b/storage/src/vespa/storage/persistence/messages.cpp
@@ -6,17 +6,14 @@ using document::BucketSpace;
namespace storage {
-GetIterCommand::GetIterCommand(framework::MemoryToken::UP token,
- const document::Bucket &bucket,
+GetIterCommand::GetIterCommand(const document::Bucket &bucket,
const spi::IteratorId iteratorId,
uint32_t maxByteSize)
: api::InternalCommand(ID),
- _token(std::move(token)),
_bucket(bucket),
_iteratorId(iteratorId),
_maxByteSize(maxByteSize)
{
- assert(_token.get());
}
GetIterCommand::~GetIterCommand() { }
@@ -38,7 +35,6 @@ GetIterCommand::makeReply() {
GetIterReply::GetIterReply(GetIterCommand& cmd)
: api::InternalReply(ID, cmd),
- _token(cmd.releaseMemoryToken()),
_bucket(cmd.getBucket()),
_completed(false)
{ }
diff --git a/storage/src/vespa/storage/persistence/messages.h b/storage/src/vespa/storage/persistence/messages.h
index 7bbeea8a12a..ba7f5979569 100644
--- a/storage/src/vespa/storage/persistence/messages.h
+++ b/storage/src/vespa/storage/persistence/messages.h
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
-#include <vespa/storageframework/generic/memory/memorytoken.h>
#include <vespa/storageapi/message/internal.h>
#include <vespa/persistence/spi/docentry.h>
#include <vespa/persistence/spi/bucket.h>
@@ -14,7 +13,6 @@ namespace storage {
class GetIterCommand : public api::InternalCommand {
private:
- mutable framework::MemoryToken::UP _token;
document::Bucket _bucket;
spi::IteratorId _iteratorId;
uint32_t _maxByteSize;
@@ -24,8 +22,7 @@ public:
typedef std::unique_ptr<GetIterCommand> UP;
typedef std::shared_ptr<GetIterCommand> SP;
- GetIterCommand(framework::MemoryToken::UP token,
- const document::Bucket &bucket,
+ GetIterCommand(const document::Bucket &bucket,
const spi::IteratorId iteratorId,
uint32_t maxByteSize);
~GetIterCommand();
@@ -44,13 +41,11 @@ public:
void print(std::ostream& out, bool verbose, const std::string& indent) const override;
private:
- framework::MemoryToken::UP releaseMemoryToken() { return std::move(_token); }
friend class GetIterReply;
};
class GetIterReply : public api::InternalReply {
private:
- framework::MemoryToken::UP _token;
document::Bucket _bucket;
std::vector<spi::DocEntry::UP> _entries;
bool _completed;
diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt
index 3d0a0fdfa14..c0238922a91 100644
--- a/storage/src/vespa/storage/storageserver/CMakeLists.txt
+++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt
@@ -11,7 +11,6 @@ vespa_add_library(storage_storageserver
documentapiconverter.cpp
fnetlistener.cpp
mergethrottler.cpp
- messageallocationtypes.cpp
messagesink.cpp
opslogger.cpp
priorityconverter.cpp
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
index eae51b90165..b214ef4539a 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
@@ -96,13 +96,6 @@ StorageTransportContext::StorageTransportContext(std::unique_ptr<RPCRequestWrapp
StorageTransportContext::~StorageTransportContext() { }
-const framework::MemoryAllocationType&
-CommunicationManager::getAllocationType(api::StorageMessage& msg) const
-{
- return _messageAllocTypes.getType(msg.getType().getId());
-}
-
-
void
CommunicationManager::receiveStorageReply(const std::shared_ptr<api::StorageReply>& reply)
{
@@ -296,8 +289,7 @@ CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, co
_configUri(configUri),
_closed(false),
_bucketResolver(std::make_unique<PlaceHolderBucketResolver>()),
- _docApiConverter(configUri, *_bucketResolver),
- _messageAllocTypes(_component.getMemoryManager())
+ _docApiConverter(configUri, *_bucketResolver)
{
_component.registerMetricUpdateHook(*this, framework::SecondTime(5));
_component.registerMetric(_metrics);
@@ -483,33 +475,9 @@ CommunicationManager::process(const std::shared_ptr<api::StorageMessage>& msg)
void
CommunicationManager::enqueue(const std::shared_ptr<api::StorageMessage> & msg)
{
- using MemoryToken = framework::MemoryToken;
assert(msg.get());
-
- const uint32_t memoryFootprint = msg->getMemoryFootprint();
- MemoryToken::UP token = _component.getMemoryManager().allocate(getAllocationType(*msg), memoryFootprint * 2,
- memoryFootprint * 2, msg->getPriority());
-
- if (token) {
- msg->setMemoryToken(std::move(token));
-
- LOG(spam, "Enq storage message %s, priority %d", msg->toString().c_str(), msg->getPriority());
- _eventQueue.enqueue(msg);
- } else {
- _metrics.failedDueToTooLittleMemory.inc();
- std::ostringstream ost;
- ost << "Failed to aquire " << (memoryFootprint * 2)
- << " bytes of memory to handle command of type "
- << msg->getType() << "\n";
- LOG(spam, "%s", ost.str().c_str());
- api::StorageCommand* cmd(dynamic_cast<api::StorageCommand*>(msg.get()));
-
- if (cmd) {
- std::shared_ptr<api::StorageReply> reply(cmd->makeReply());
- reply->setResult(api::ReturnCode(api::ReturnCode::BUSY, ost.str()));
- sendReply(reply);
- }
- }
+ LOG(spam, "Enq storage message %s, priority %d", msg->toString().c_str(), msg->getPriority());
+ _eventQueue.enqueue(msg);
}
bool
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h
index 4cf3f33e6ea..89b76583cb5 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.h
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.h
@@ -11,7 +11,6 @@
#pragma once
#include "communicationmanagermetrics.h"
-#include "messageallocationtypes.h"
#include "documentapiconverter.h"
#include <vespa/storage/common/storagelink.h>
#include <vespa/storage/common/storagecomponent.h>
@@ -171,10 +170,7 @@ private:
std::unique_ptr<BucketResolver> _bucketResolver;
DocumentApiConverter _docApiConverter;
framework::Thread::UP _thread;
- MessageAllocationTypes _messageAllocTypes;
- const framework::MemoryAllocationType&
- getAllocationType(api::StorageMessage& msg) const;
void updateMetrics(const MetricLockGuard &) override;
// Test needs access to configure() for live reconfig testing.
diff --git a/storage/src/vespa/storage/storageserver/framework.h b/storage/src/vespa/storage/storageserver/framework.h
index 724b4fe0944..f0ea1d71aa6 100644
--- a/storage/src/vespa/storage/storageserver/framework.h
+++ b/storage/src/vespa/storage/storageserver/framework.h
@@ -18,7 +18,6 @@
#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h>
#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h>
namespace storage {
@@ -27,7 +26,6 @@ struct Framework {
// Typedefs to simplify the remainder of the interface
typedef StorageComponentRegisterImpl CompReg;
typedef framework::defaultimplementation::RealClock RealClock;
- typedef framework::defaultimplementation::MemoryManager MemoryManager;
/**
* You can provide your own clock implementation. Useful in testing where
@@ -48,20 +46,10 @@ struct Framework {
*/
FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); }
- /**
- * Get the memory manager. Components that wants to print status of memory
- * manager need access to the actual implementation.
- */
- MemoryManager& getMemoryManager() { return _memoryManager; }
-
- void setMaximumMemoryUsage(uint64_t max);
-
private:
CompReg _componentRegister;
framework::Clock::UP _clock;
framework::defaultimplementation::ThreadPoolImpl _threadPool;
- framework::defaultimplementation::AllocationLogic* _memoryLogic;
- MemoryManager _memoryManager;
};
diff --git a/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp b/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp
deleted file mode 100644
index ac2d99a439c..00000000000
--- a/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "messageallocationtypes.h"
-#include <vespa/storageapi/messageapi/storagemessage.h>
-#include <vespa/vespalib/util/exceptions.h>
-#include <vespa/vespalib/stllike/asciistream.h>
-
-namespace storage {
-
-MessageAllocationTypes::MessageAllocationTypes(framework::MemoryManagerInterface& manager)
-{
- using api::MessageType;
- using framework::MemoryAllocationType;
-
- _types.resize(MessageType::MESSAGETYPE_MAX_ID);
- _types[MessageType::DOCBLOCK_ID] = &manager.registerAllocationType(MemoryAllocationType("MESSAGE_DOCBLOCK"));
- _types[MessageType::DOCBLOCK_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MESSAGE_DOCBLOCK_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GET", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::GET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::INTERNAL_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::INTERNAL"));
- _types[MessageType::INTERNAL_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::INTERNAL_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::PUT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::PUT", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::PUT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::PUT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::REMOVE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVE", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::REMOVE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::REVERT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REVERT"));
- _types[MessageType::REVERT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REVERT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::VISITOR_CREATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_CREATE", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::VISITOR_CREATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_CREATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::VISITOR_DESTROY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_DESTROY"));
- _types[MessageType::VISITOR_DESTROY_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_DESTROY_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::REQUESTBUCKETINFO_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REQUESTBUCKETINFO"));
- _types[MessageType::REQUESTBUCKETINFO_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REQUESTBUCKETINFO_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::NOTIFYBUCKETCHANGE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::NOTIFYBUCKETCHANGE"));
- _types[MessageType::NOTIFYBUCKETCHANGE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::NOTIFYBUCKETCHANGE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::CREATEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::CREATEBUCKET"));
- _types[MessageType::CREATEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::CREATEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::MERGEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MERGEBUCKET"));
- _types[MessageType::MERGEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MERGEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::DELETEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DELETEBUCKET"));
- _types[MessageType::DELETEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DELETEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SETNODESTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETNODESTATE", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SETNODESTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETNODESTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETNODESTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETNODESTATE", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETNODESTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETNODESTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SETSYSTEMSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETSYSTEMSTATE", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SETSYSTEMSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETSYSTEMSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETSYSTEMSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETSYSTEMSTATE", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETSYSTEMSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETSYSTEMSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETBUCKETDIFF_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETDIFF", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETBUCKETDIFF_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETDIFF_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::APPLYBUCKETDIFF_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::APPLYBUCKETDIFF", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::APPLYBUCKETDIFF_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::APPLYBUCKETDIFF_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::VISITOR_INFO_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_INFO"));
- _types[MessageType::VISITOR_INFO_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_INFO_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SEARCHRESULT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SEARCHRESULT"));
- _types[MessageType::SEARCHRESULT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SEARCHRESULT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SPLITBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SPLITBUCKET"));
- _types[MessageType::SPLITBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SPLITBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::JOINBUCKETS_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::JOINBUCKETS"));
- _types[MessageType::JOINBUCKETS_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::JOINBUCKETS_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SETBUCKETSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETBUCKETSTATE"));
- _types[MessageType::SETBUCKETSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETBUCKETSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::MULTIOPERATION_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MULTIOPERATION", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::MULTIOPERATION_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MULTIOPERATION_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::DOCUMENTSUMMARY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTSUMMARY"));
- _types[MessageType::DOCUMENTSUMMARY_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTSUMMARY_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::MAPVISITOR_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MAPVISITOR"));
- _types[MessageType::MAPVISITOR_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MAPVISITOR_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::STATBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::STATBUCKET", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::STATBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::STATBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETBUCKETLIST_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETLIST", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::GETBUCKETLIST_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETLIST_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::DOCUMENTLIST_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTLIST"));
- _types[MessageType::DOCUMENTLIST_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTLIST_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::UPDATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::UPDATE", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::UPDATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::UPDATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::EMPTYBUCKETS_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::EMPTYBUCKETS"));
- _types[MessageType::EMPTYBUCKETS_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::EMPTYBUCKETS_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::REMOVELOCATION_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVELOCATION", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::REMOVELOCATION_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVELOCATION_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::QUERYRESULT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::QUERYRESULT"));
- _types[MessageType::QUERYRESULT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::QUERYRESULT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::BATCHPUTREMOVE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHPUTREMOVE", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::BATCHPUTREMOVE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHPUTREMOVE_REPLY", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::BATCHDOCUMENTUPDATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHDOCUMENTUPDATE", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::BATCHDOCUMENTUPDATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHDOCUMENTUPDATE_REPLY", framework::MemoryAllocationType::EXTERNAL_LOAD));
-}
-
-const framework::MemoryAllocationType&
-MessageAllocationTypes::getType(uint32_t type) const {
- if (_types.size() > size_t(type) && _types[type] != 0) {
- return *_types[type];
- }
- vespalib::asciistream ost;
- ost << "No type registered with value " << type << ".";
- throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
-}
-
-} // storage
diff --git a/storage/src/vespa/storage/storageserver/messageallocationtypes.h b/storage/src/vespa/storage/storageserver/messageallocationtypes.h
deleted file mode 100644
index 8f7e8e4b3c3..00000000000
--- a/storage/src/vespa/storage/storageserver/messageallocationtypes.h
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::MessageAllocationTypes
- *
- * \brief Memory allocation types for messages in storage.
- */
-#pragma once
-
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
-#include <vector>
-
-namespace storage {
-
-class MessageAllocationTypes {
- std::vector<const framework::MemoryAllocationType*> _types;
-
-public:
- MessageAllocationTypes(framework::MemoryManagerInterface& manager);
-
- const framework::MemoryAllocationType& getType(uint32_t type) const;
-};
-
-}
-
diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp
index 855efaed6aa..ba1556bd3b9 100644
--- a/storage/src/vespa/storage/storageserver/storagenode.cpp
+++ b/storage/src/vespa/storage/storageserver/storagenode.cpp
@@ -7,7 +7,6 @@
#include "storagemetricsset.h"
#include "storagenodecontext.h"
-#include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h>
#include <vespa/storage/frameworkimpl/status/statuswebserver.h>
#include <vespa/storage/frameworkimpl/thread/deadlockdetector.h>
#include <vespa/storage/common/statusmetricconsumer.h>
@@ -117,7 +116,6 @@ StorageNode::initialize()
// and store them away, while having the config lock.
subscribeToConfigs();
- _context.getMemoryManager().setMaximumMemoryUsage(_serverConfig->memorytouse);
updateUpgradeFlag(*_clusterConfig);
// First update some basics that doesn't depend on anything else to be
@@ -157,11 +155,6 @@ StorageNode::initialize()
initializeNodeSpecific();
- _memoryStatusViewer.reset(new MemoryStatusViewer(
- _context.getMemoryManager(),
- _context.getComponentRegister().getMetricManager(),
- _context.getComponentRegister()));
-
_statusMetrics.reset(new StatusMetricConsumer(
_context.getComponentRegister(), _context.getComponentRegister().getMetricManager()));
_stateReporter.reset(new StateReporter(
@@ -259,14 +252,6 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard)
DIFFERWARN(clusterName, "Cannot alter cluster name of node live");
DIFFERWARN(nodeIndex, "Cannot alter node index of node live");
DIFFERWARN(isDistributor, "Cannot alter role of node live");
- {
- if (DIFFER(memorytouse)) {
- LOG(info, "Live config update: Memory to use changed from %" PRId64 " to %" PRId64 ".",
- oldC.memorytouse, newC.memorytouse);
- ASSIGN(memorytouse);
- _context.getMemoryManager().setMaximumMemoryUsage(newC.memorytouse);
- }
- }
_serverConfig.reset(new StorServerConfig(oldC));
_newServerConfig.reset();
(void)updated;
@@ -413,10 +398,6 @@ StorageNode::shutdown()
LOG(debug, "Deleting state reporter");
_stateReporter.reset();
}
- if (_memoryStatusViewer) {
- LOG(debug, "Deleting memory status viewer");
- _memoryStatusViewer.reset();
- }
if (_stateManager) {
LOG(debug, "Deleting state manager");
_stateManager.reset();
diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h
index 9b727ef3e0c..e9d3004be68 100644
--- a/storage/src/vespa/storage/storageserver/storagenode.h
+++ b/storage/src/vespa/storage/storageserver/storagenode.h
@@ -121,8 +121,6 @@ private:
// Depends on bucket databases and stop() functionality
std::unique_ptr<DeadLockDetector> _deadLockDetector;
- // Depends on dead lock detector and threadpool
- std::unique_ptr<MemoryStatusViewer> _memoryStatusViewer;
// Depends on metric manager
std::unique_ptr<StatusMetricConsumer> _statusMetrics;
// Depends on metric manager
diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp
index 2e1aa52e68d..75e1f12773f 100644
--- a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp
+++ b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp
@@ -2,31 +2,15 @@
#include "storagenodecontext.h"
-#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h>
-
namespace storage {
-using framework::defaultimplementation::AllocationLogic;
-using framework::defaultimplementation::PriorityMemoryLogic;
-
StorageNodeContext::StorageNodeContext(ComponentRegister::UP compReg, framework::Clock::UP clock)
: _componentRegister(std::move(compReg)),
_clock(std::move(clock)),
- _threadPool(*_clock),
- _memoryLogic(new PriorityMemoryLogic(*_clock, 1024 * 1024 * 1024)),
- _memoryManager(AllocationLogic::UP(_memoryLogic))
+ _threadPool(*_clock)
{
_componentRegister->setClock(*_clock);
_componentRegister->setThreadPool(_threadPool);
- _componentRegister->setMemoryManager(_memoryManager);
-}
-
-void
-StorageNodeContext::setMaximumMemoryUsage(uint64_t max)
-{
- using storage::framework::defaultimplementation::PriorityMemoryLogic;
- dynamic_cast<PriorityMemoryLogic*>(_memoryLogic)
- ->setMaximumMemoryUsage(max);
}
} // storage
diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.h b/storage/src/vespa/storage/storageserver/storagenodecontext.h
index 0149f975f63..eabca618bfb 100644
--- a/storage/src/vespa/storage/storageserver/storagenodecontext.h
+++ b/storage/src/vespa/storage/storageserver/storagenodecontext.h
@@ -18,7 +18,6 @@
#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h>
#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h>
namespace storage {
@@ -27,7 +26,6 @@ struct StorageNodeContext {
// Typedefs to simplify the remainder of the interface
typedef StorageComponentRegisterImpl ComponentRegister;
typedef framework::defaultimplementation::RealClock RealClock;
- typedef framework::defaultimplementation::MemoryManager MemoryManager;
/**
* Get the actual component register. Available as the actual type as the
@@ -42,14 +40,6 @@ struct StorageNodeContext {
*/
FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); }
- /**
- * Get the memory manager. Components that wants to print status of memory
- * manager need access to the actual implementation.
- */
- MemoryManager& getMemoryManager() { return _memoryManager; }
-
- void setMaximumMemoryUsage(uint64_t max);
-
protected:
// Initialization has been split in two as subclass needs to initialize
// component register before sending it on.
@@ -59,8 +49,6 @@ private:
ComponentRegister::UP _componentRegister;
framework::Clock::UP _clock;
framework::defaultimplementation::ThreadPoolImpl _threadPool;
- framework::defaultimplementation::AllocationLogic* _memoryLogic;
- MemoryManager _memoryManager;
};
diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp
index b91e38e035e..0d230745875 100644
--- a/storage/src/vespa/storage/visiting/visitor.cpp
+++ b/storage/src/vespa/storage/visiting/visitor.cpp
@@ -2,7 +2,6 @@
#include "visitor.h"
#include "visitormetrics.h"
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/generic/clock/timer.h>
#include <vespa/storageapi/message/datagram.h>
#include <vespa/storage/persistence/messages.h>
@@ -267,8 +266,7 @@ Visitor::Visitor(StorageComponent& component)
_id(),
_controlDestination(),
_dataDestination(),
- _documentSelection(),
- _memoryManager(0)
+ _documentSelection()
{
}
@@ -603,10 +601,6 @@ Visitor::start(api::VisitorId id, api::StorageMessage::Id cmdId,
_documentPriority = documentPriority;
_state = STATE_RUNNING;
- if (_memoryAllocType == 0) {
- _memoryAllocType = &_component.getMemoryManager()
- .getAllocationType("VISITOR_BUFFER");
- }
LOG(debug, "Starting visitor '%s' for %" PRIu64 " buckets from %" PRIu64 " to "
"%" PRIu64 ". First is %s. Max pending replies: %u, include "
@@ -803,17 +797,7 @@ Visitor::onCreateIteratorReply(
LOG(debug, "Visitor '%s' starting to visit bucket %s.",
_id.c_str(), bucketId.toString().c_str());
- framework::MemoryToken::UP token(
- _memoryManager->allocate(
- *_memoryAllocType, _docBlockSize, _docBlockSize, _priority));
- if (token.get() == 0) {
- // Not enough memory
- return;
- }
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), bucket,
- bucketState.getIteratorId(),
- _docBlockSize));
+ auto cmd = std::make_shared<GetIterCommand>(bucket, bucketState.getIteratorId(), _docBlockSize);
cmd->setLoadType(_initiatingCmd->getLoadType());
cmd->getTrace().setLevel(_traceLevel);
cmd->setPriority(_priority);
@@ -1223,19 +1207,8 @@ Visitor::getIterators()
it = _bucketStates.erase(it);
continue;
}
- framework::MemoryToken::UP token(
- _memoryManager->allocate(
- *_memoryAllocType, _docBlockSize, _docBlockSize,
- _priority));
- if (token.get() == 0) {
- // Not enough memory
- return true;
- }
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token),
- bucketState.getBucket(),
- bucketState.getIteratorId(),
- _docBlockSize));
+ auto cmd = std::make_shared<GetIterCommand>(
+ bucketState.getBucket(), bucketState.getIteratorId(), _docBlockSize);
cmd->setLoadType(_initiatingCmd->getLoadType());
cmd->getTrace().setLevel(_traceLevel);
cmd->setPriority(_priority);
diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h
index 4436312032f..f84b105f1ed 100644
--- a/storage/src/vespa/storage/visiting/visitor.h
+++ b/storage/src/vespa/storage/visiting/visitor.h
@@ -327,7 +327,6 @@ private:
protected:
// These variables should not be altered after visitor starts. This not
// controlled by locks.
- const framework::MemoryAllocationType* _memoryAllocType;
VisitorMessageHandler* _messageHandler;
VisitorMessageSession::UP _messageSession;
documentapi::Priority::Value _documentPriority;
@@ -339,7 +338,6 @@ protected:
std::string _documentSelectionString;
std::unique_ptr<document::OrderingSpecification> _ordering;
vdslib::VisitorStatistics _visitorStatistics;
- framework::MemoryManagerInterface* _memoryManager;
bool isCompletedCalled() const { return _calledCompletedVisitor; }
@@ -365,8 +363,6 @@ public:
const api::StorageMessageAddress* getDataDestination() const
{ return _dataDestination.get(); } // Can't be null if attached
- void setAllocationType(const framework::MemoryAllocationType& mat)
- { _memoryAllocType = &mat; }
void setMaxPending(unsigned int maxPending)
{ _visitorOptions._maxPending = maxPending; }
@@ -384,8 +380,6 @@ public:
{ _docBlockTimeout = timeout; }
void setVisitorInfoTimeout(framework::MilliSecTime timeout)
{ _visitorInfoTimeout = timeout; }
- void setMemoryManager(framework::MemoryManagerInterface& mm)
- { _memoryManager = &mm; }
void setOwnNodeIndex(uint16_t nodeIndex) { _ownNodeIndex = nodeIndex; }
void setBucketSpace(document::BucketSpace bucketSpace) { _bucketSpace = bucketSpace; }
diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp
index deb38bb927d..f207c4436b1 100644
--- a/storage/src/vespa/storage/visiting/visitormanager.cpp
+++ b/storage/src/vespa/storage/visiting/visitormanager.cpp
@@ -7,7 +7,6 @@
#include "countvisitor.h"
#include "testvisitor.h"
#include "recoveryvisitor.h"
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storage/common/statusmessages.h>
#include <vespa/config/common/exceptions.h>
#include <vespa/documentapi/loadtypes/loadtypeset.h>
@@ -46,10 +45,6 @@ VisitorManager::VisitorManager(const config::ConfigUri & configUri,
_enforceQueueUse(false),
_visitorFactories(externalFactories)
{
- _component.getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType(
- "VISITOR_BUFFER",
- framework::MemoryAllocationType::EXTERNAL_LOAD));
_configFetcher.subscribe<vespa::config::content::core::StorVisitorConfig>(configUri.getConfigId(), this);
_configFetcher.start();
_component.registerMetric(*_metrics);
diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp
index d3fed86b741..4743ac66054 100644
--- a/storage/src/vespa/storage/visiting/visitorthread.cpp
+++ b/storage/src/vespa/storage/visiting/visitorthread.cpp
@@ -2,7 +2,6 @@
#include "visitorthread.h"
#include "messages.h"
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/document/repo/documenttyperepo.h>
#include <vespa/document/select/bodyfielddetector.h>
#include <vespa/document/select/orderingselector.h>
@@ -100,9 +99,7 @@ VisitorThread::VisitorThread(uint32_t threadIndex,
_timeBetweenTicks(1000),
_component(componentRegister, getThreadName(threadIndex)),
_messageSessionFactory(messageSessionFac),
- _visitorFactories(visitorFactories),
- _memoryBufferAlloc(
- _component.getMemoryManager().getAllocationType("VISITOR_BUFFER"))
+ _visitorFactories(visitorFactories)
{
framework::MilliSecTime maxProcessingTime(30 * 1000);
framework::MilliSecTime waitTime(1000);
@@ -473,8 +470,6 @@ VisitorThread::onCreateVisitor(
cmd->getInstanceId().c_str(), errors.str().c_str());
break;
}
- visitor->setAllocationType(_memoryBufferAlloc);
- visitor->setMemoryManager(_component.getMemoryManager());
// Set visitor parameters
if (cmd->getMaximumPendingReplyCount() != 0) {
visitor->setMaxPending(cmd->getMaximumPendingReplyCount());
diff --git a/storage/src/vespa/storage/visiting/visitorthread.h b/storage/src/vespa/storage/visiting/visitorthread.h
index 1f1459ccb2b..97ca70f3761 100644
--- a/storage/src/vespa/storage/visiting/visitorthread.h
+++ b/storage/src/vespa/storage/visiting/visitorthread.h
@@ -90,7 +90,6 @@ class VisitorThread : public framework::Runnable,
framework::Thread::UP _thread;
VisitorMessageSessionFactory& _messageSessionFactory;
VisitorFactory::Map& _visitorFactories;
- const framework::MemoryAllocationType& _memoryBufferAlloc;
public:
VisitorThread(uint32_t threadIndex,