summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTor Brede Vekterli <vekterli@yahoo-inc.com>2017-12-04 11:31:29 +0000
committerTor Brede Vekterli <vekterli@yahoo-inc.com>2017-12-04 13:06:46 +0000
commit952a877611d657cfa0166b14699c8731b18f7587 (patch)
treec1b4330b9dc512c3ffbf845e626857ddcd801982
parentd164fbb93e277ef23ab610320a7cf8556e3c036e (diff)
Remove memory manager component from content layer
We already have resource utilization tracking in both MessageBus and the search core. The memory manager has never been auto-scaled based on the hardware present and adds a _lot_ of complexity without having any known instances where it has actually saved the day. Removing it also removes a mutex on the message hot path. If we need such functionality in the future, should design a lock-free solution. Cleanup
-rw-r--r--memfilepersistence/src/tests/spi/memcachetest.cpp11
-rw-r--r--memfilepersistence/src/tests/spi/memfiletestutils.cpp7
-rw-r--r--memfilepersistence/src/tests/spi/memfiletestutils.h4
-rw-r--r--memfilepersistence/src/tests/spi/providerconformancetest.cpp9
-rw-r--r--memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp9
-rw-r--r--memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h6
-rw-r--r--memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp3
-rw-r--r--storage/CMakeLists.txt2
-rw-r--r--storage/src/tests/CMakeLists.txt1
-rw-r--r--storage/src/tests/common/teststorageapp.h1
-rw-r--r--storage/src/tests/frameworkimpl/memory/CMakeLists.txt8
-rw-r--r--storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp170
-rw-r--r--storage/src/tests/persistence/filestorage/filestormanagertest.cpp57
-rw-r--r--storage/src/tests/persistence/persistencetestutils.cpp1
-rw-r--r--storage/src/tests/persistence/persistencetestutils.h1
-rw-r--r--storage/src/tests/storageserver/communicationmanagertest.cpp1
-rw-r--r--storage/src/vespa/storage/CMakeLists.txt1
-rw-r--r--storage/src/vespa/storage/bucketdb/bucketmanager.cpp9
-rw-r--r--storage/src/vespa/storage/bucketdb/bucketmanager.h2
-rw-r--r--storage/src/vespa/storage/common/statusmetricconsumer.cpp15
-rw-r--r--storage/src/vespa/storage/common/statusmetricconsumer.h1
-rw-r--r--storage/src/vespa/storage/distributor/bucketdbupdater.h1
-rw-r--r--storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h2
-rw-r--r--storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt8
-rw-r--r--storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h20
-rw-r--r--storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp661
-rw-r--r--storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h137
-rw-r--r--storage/src/vespa/storage/persistence/messages.cpp6
-rw-r--r--storage/src/vespa/storage/persistence/messages.h7
-rw-r--r--storage/src/vespa/storage/storageserver/CMakeLists.txt1
-rw-r--r--storage/src/vespa/storage/storageserver/communicationmanager.cpp38
-rw-r--r--storage/src/vespa/storage/storageserver/communicationmanager.h4
-rw-r--r--storage/src/vespa/storage/storageserver/framework.h12
-rw-r--r--storage/src/vespa/storage/storageserver/messageallocationtypes.cpp100
-rw-r--r--storage/src/vespa/storage/storageserver/messageallocationtypes.h24
-rw-r--r--storage/src/vespa/storage/storageserver/storagenode.cpp19
-rw-r--r--storage/src/vespa/storage/storageserver/storagenode.h2
-rw-r--r--storage/src/vespa/storage/storageserver/storagenodecontext.cpp18
-rw-r--r--storage/src/vespa/storage/storageserver/storagenodecontext.h12
-rw-r--r--storage/src/vespa/storage/visiting/visitor.cpp35
-rw-r--r--storage/src/vespa/storage/visiting/visitor.h6
-rw-r--r--storage/src/vespa/storage/visiting/visitormanager.cpp5
-rw-r--r--storage/src/vespa/storage/visiting/visitorthread.cpp7
-rw-r--r--storage/src/vespa/storage/visiting/visitorthread.h1
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/storagemessage.h6
-rw-r--r--storageframework/CMakeLists.txt3
-rw-r--r--storageframework/src/tests/CMakeLists.txt1
-rw-r--r--storageframework/src/tests/memory/.gitignore2
-rw-r--r--storageframework/src/tests/memory/CMakeLists.txt8
-rw-r--r--storageframework/src/tests/memory/memorymanagertest.cpp397
-rw-r--r--storageframework/src/tests/memory/memorystatetest.cpp172
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt1
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp10
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h1
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp6
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h3
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore2
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt10
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h48
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp157
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h156
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp225
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h139
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp46
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h66
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp30
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h29
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp232
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h98
-rw-r--r--storageframework/src/vespa/storageframework/generic/CMakeLists.txt1
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/.gitignore2
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt6
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h51
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h63
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp13
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/memorytoken.h29
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h41
-rw-r--r--storageframework/src/vespa/storageframework/storageframework.h1
78 files changed, 26 insertions, 3474 deletions
diff --git a/memfilepersistence/src/tests/spi/memcachetest.cpp b/memfilepersistence/src/tests/spi/memcachetest.cpp
index 73815df563a..5e9f1a28225 100644
--- a/memfilepersistence/src/tests/spi/memcachetest.cpp
+++ b/memfilepersistence/src/tests/spi/memcachetest.cpp
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include <vespa/memfilepersistence/memfile/memfilecache.h>
-#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h>
#include <tests/spi/memfiletestutils.h>
@@ -41,7 +40,6 @@ private:
framework::defaultimplementation::ComponentRegisterImpl::UP _register;
framework::Component::UP _component;
FakeClock::UP _clock;
- framework::defaultimplementation::MemoryManager::UP _memoryManager;
std::unique_ptr<MemFilePersistenceMetrics> _metrics;
std::unique_ptr<MemFileCache> _cache;
@@ -113,18 +111,10 @@ private:
new framework::defaultimplementation::ComponentRegisterImpl);
_clock.reset(new FakeClock);
_register->setClock(*_clock);
- _memoryManager.reset(
- new framework::defaultimplementation::MemoryManager(
- framework::defaultimplementation::AllocationLogic::UP(
- new framework::defaultimplementation::SimpleMemoryLogic(
- *_clock, maxMemory * 2))));
- _register->setMemoryManager(*_memoryManager);
_component.reset(new framework::Component(*_register, "testcomponent"));
_metrics.reset(new MemFilePersistenceMetrics(*_component));
_cache.reset(new MemFileCache(*_register, _metrics->_cache));
setCacheSize(maxMemory);
- _memoryManager->registerAllocationType(framework::MemoryAllocationType(
- "steal", framework::MemoryAllocationType::FORCE_ALLOCATE));
}
public:
@@ -133,7 +123,6 @@ public:
_metrics.reset(0);
_component.reset(0);
_register.reset(0);
- _memoryManager.reset(0);
_clock.reset(0);
}
};
diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.cpp b/memfilepersistence/src/tests/spi/memfiletestutils.cpp
index bb35aa4d03e..cd14e989220 100644
--- a/memfilepersistence/src/tests/spi/memfiletestutils.cpp
+++ b/memfilepersistence/src/tests/spi/memfiletestutils.cpp
@@ -5,7 +5,6 @@
#include <tests/spi/memfiletestutils.h>
#include <tests/spi/simulatedfailurefile.h>
#include <vespa/memfilepersistence/memfile/memfilecache.h>
-#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h>
#include <vespa/document/update/assignvalueupdate.h>
#include <vespa/document/test/make_bucket_space.h>
#include <vespa/persistence/spi/test.h>
@@ -72,12 +71,6 @@ MemFileTestUtils::setupDisks(uint32_t numDisks) {
new framework::defaultimplementation::ComponentRegisterImpl);
_clock.reset(new FakeClock);
_componentRegister->setClock(*_clock);
- _memoryManager.reset(
- new framework::defaultimplementation::MemoryManager(
- framework::defaultimplementation::AllocationLogic::UP(
- new framework::defaultimplementation::SimpleMemoryLogic(
- *_clock, 1024 * 1024 * 1024))));
- _componentRegister->setMemoryManager(*_memoryManager);
_env.reset(new MemFileTestEnvironment(numDisks,
*_componentRegister,
*getTypeRepo()));
diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.h b/memfilepersistence/src/tests/spi/memfiletestutils.h
index dde73c88164..450d87451b9 100644
--- a/memfilepersistence/src/tests/spi/memfiletestutils.h
+++ b/memfilepersistence/src/tests/spi/memfiletestutils.h
@@ -20,8 +20,6 @@
#include <vespa/document/update/documentupdate.h>
#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
#include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
-
namespace storage {
namespace memfile {
@@ -70,7 +68,6 @@ private:
document::BucketIdFactory _bucketIdFactory;
framework::defaultimplementation::ComponentRegisterImpl::UP _componentRegister;
FakeClock::UP _clock;
- framework::defaultimplementation::MemoryManager::UP _memoryManager;
std::unique_ptr<MemFileTestEnvironment> _env;
public:
@@ -82,7 +79,6 @@ public:
void tearDown() override{
_env.reset();
_componentRegister.reset();
- _memoryManager.reset();
_clock.reset();
}
diff --git a/memfilepersistence/src/tests/spi/providerconformancetest.cpp b/memfilepersistence/src/tests/spi/providerconformancetest.cpp
index db02e619782..625549f0008 100644
--- a/memfilepersistence/src/tests/spi/providerconformancetest.cpp
+++ b/memfilepersistence/src/tests/spi/providerconformancetest.cpp
@@ -2,7 +2,6 @@
#include "memfiletestutils.h"
#include <vespa/persistence/conformancetest/conformancetest.h>
-#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h>
namespace storage {
namespace memfile {
@@ -11,19 +10,13 @@ struct ProviderConformanceTest : public spi::ConformanceTest {
struct Factory : public PersistenceFactory {
framework::defaultimplementation::ComponentRegisterImpl _compRegister;
framework::defaultimplementation::RealClock _clock;
- framework::defaultimplementation::MemoryManager _memoryManager;
std::unique_ptr<MemFileCache> cache;
Factory()
: _compRegister(),
- _clock(),
- _memoryManager(
- framework::defaultimplementation::AllocationLogic::UP(
- new framework::defaultimplementation::SimpleMemoryLogic(
- _clock, 1024 * 1024 * 1024)))
+ _clock()
{
_compRegister.setClock(_clock);
- _compRegister.setMemoryManager(_memoryManager);
}
spi::PersistenceProvider::UP
diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp
index 5f24d28a309..cfd73d1a7cb 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp
@@ -31,7 +31,7 @@ MemFileCache::returnToCache(MemFileCache::Entry& entry)
MemoryUsage newUsage = entry._file.getCacheSize();
- if (_memoryToken->getSize() == 0 || newUsage.sum() == 0) {
+ if (_cacheLimit.sum() == 0 || newUsage.sum() == 0) {
entry._file.flushToDisk();
eraseNoLock(id);
return;
@@ -155,10 +155,6 @@ MemFileCache::MemFileCache(framework::ComponentRegister& componentRegister,
MemFilePersistenceCacheMetrics& metrics)
: Component(componentRegister, "memfilecache"),
_lastUsedCounter(0),
- _allocationType(getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType(
- "memfilecache", framework::MemoryAllocationType::CACHE))),
- _memoryToken(getMemoryManager().allocate(_allocationType, 0, 0, 200)),
_metrics(metrics),
_bodyEvicter(_metrics.body_evictions),
_headerEvicter(_metrics.header_evictions),
@@ -172,7 +168,6 @@ MemFileCache::setCacheSize(MemoryUsage cacheSize)
vespalib::LockGuard lock(_cacheLock);
_cacheLimit = cacheSize;
- _memoryToken->resize(std::min(_memoryToken->getSize(), _cacheLimit.sum()), _cacheLimit.sum());
evictWhileFull();
}
@@ -491,7 +486,7 @@ MemFileCache::Statistics
MemFileCache::getCacheStats() const
{
vespalib::LockGuard lock(_cacheLock);
- return Statistics(_memoryUsage, _memoryToken->getSize(), _entries.size());
+ return Statistics(_memoryUsage, _cacheLimit.sum(), _entries.size());
}
void
diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h
index e45f8f7f1f6..36e6e6b641a 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h
+++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h
@@ -21,7 +21,6 @@
#include <boost/multi_index/mem_fun.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/sequenced_index.hpp>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/generic/component/component.h>
@@ -191,8 +190,6 @@ private:
LRUCache _entries;
uint64_t _lastUsedCounter;
- const framework::MemoryAllocationType& _allocationType;
- framework::MemoryToken::UP _memoryToken;
MemFilePersistenceCacheMetrics& _metrics;
@@ -273,7 +270,6 @@ public:
/**
* Used for unit testing only.
*/
- framework::MemoryToken& getMemoryToken() { return *_memoryToken; }
const MemFilePersistenceCacheMetrics& getMetrics() const {
return _metrics;
}
@@ -283,7 +279,7 @@ public:
*/
void setCacheSize(MemoryUsage limits);
- uint64_t getCacheSize() { return _memoryToken->getSize(); }
+ uint64_t getCacheSize() { return _cacheLimit.sum(); }
/**
* NOTE: takes lock, never call from within memfilecache code.
diff --git a/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp b/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp
index 1dccd255ebb..9781c28823d 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp
@@ -11,7 +11,6 @@
#include <vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h>
#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
#include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h>
-#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h>
#include <vespa/vespalib/util/programoptions.h>
#include <vespa/config/helper/configgetter.hpp>
#include <vespa/config/subscription/configuri.h>
@@ -139,7 +138,6 @@ namespace {
framework::defaultimplementation::ComponentRegisterImpl _compReg;
framework::Component _component;
framework::defaultimplementation::RealClock _clock;
- framework::defaultimplementation::NoMemoryManager _memoryMan;
MemFilePersistenceMetrics _metrics;
MemFilePersistenceThreadMetrics* _threadMetrics;
std::unique_ptr<MemFileCache> _cache;
@@ -177,7 +175,6 @@ namespace {
_docType("foo", 1)
{
_compReg.setClock(_clock);
- _compReg.setMemoryManager(_memoryMan);
_cache.reset(new MemFileCache(_compReg, _metrics._cache));
if (documentConfigId == 0) {
_repo.reset(new DocumentTypeRepo(_docType));
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 24d5a2d27f2..5c24d4adb3b 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -35,7 +35,6 @@ vespa_define_module(
src/vespa/storage/distributor/operations/external
src/vespa/storage/distributor/operations/idealstate
src/vespa/storage/frameworkimpl/component
- src/vespa/storage/frameworkimpl/memory
src/vespa/storage/frameworkimpl/status
src/vespa/storage/frameworkimpl/thread
src/vespa/storage/persistence
@@ -60,7 +59,6 @@ vespa_define_module(
src/tests/common
src/tests/common/hostreporter
src/tests/distributor
- src/tests/frameworkimpl/memory
src/tests/frameworkimpl/status
src/tests/persistence
src/tests/persistence/common
diff --git a/storage/src/tests/CMakeLists.txt b/storage/src/tests/CMakeLists.txt
index 7747e8bacc1..9962949edde 100644
--- a/storage/src/tests/CMakeLists.txt
+++ b/storage/src/tests/CMakeLists.txt
@@ -13,7 +13,6 @@ vespa_add_executable(storage_testrunner_app TEST
storage_testdistributor
storage_testpersistence
storage_testfilestorage
- storage_testmemory
storage_teststatus
)
diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h
index 6906d41ac47..46744ecb3c6 100644
--- a/storage/src/tests/common/teststorageapp.h
+++ b/storage/src/tests/common/teststorageapp.h
@@ -24,7 +24,6 @@
#include <vespa/storage/storageserver/framework.h>
#include <vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h>
#include <vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
#include <vespa/storageframework/defaultimplementation/component/testcomponentregister.h>
#include <vespa/persistence/spi/persistenceprovider.h>
diff --git a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt b/storage/src/tests/frameworkimpl/memory/CMakeLists.txt
deleted file mode 100644
index e5a364e02a3..00000000000
--- a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(storage_testmemory TEST
- SOURCES
- memorystatusviewertest.cpp
- DEPENDS
- storage
- storage_testcommon
-)
diff --git a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp b/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp
deleted file mode 100644
index 1ad4b16751d..00000000000
--- a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp
+++ /dev/null
@@ -1,170 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/metrics/metrics.h>
-#include <vespa/metrics/metricmanager.h>
-#include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h>
-#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h>
-#include <tests/common/teststorageapp.h>
-#include <vespa/vdstestlib/cppunit/macros.h>
-#include <vespa/vespalib/util/exceptions.h>
-#include <boost/lexical_cast.hpp>
-
-namespace storage {
-
-struct MemoryStatusViewerTest : public CppUnit::TestFixture
-{
- static const int maxMemory = 1000;
- std::unique_ptr<TestServiceLayerApp> _node;
- std::unique_ptr<framework::defaultimplementation::MemoryManager> _memMan;
-
- void setUp() override;
-
- void testEmptyState();
- void testSnapshots();
-
- CPPUNIT_TEST_SUITE(MemoryStatusViewerTest);
- CPPUNIT_TEST(testEmptyState);
- CPPUNIT_TEST(testSnapshots);
- CPPUNIT_TEST_SUITE_END();
-};
-
-CPPUNIT_TEST_SUITE_REGISTRATION(MemoryStatusViewerTest);
-
-void
-MemoryStatusViewerTest::setUp()
-{
- _node.reset(new TestServiceLayerApp(DiskCount(2)));
- framework::defaultimplementation::PriorityMemoryLogic* logic(
- new framework::defaultimplementation::PriorityMemoryLogic(
- _node->getClock(), maxMemory));
- logic->setMinJumpToUpdateMax(1);
- _memMan.reset(new framework::defaultimplementation::MemoryManager(
- framework::defaultimplementation::AllocationLogic::UP(logic)));
-}
-
-void
-MemoryStatusViewerTest::testEmptyState()
-{
- // Add a memory manager, and add a bit of load to it, so it's not
- // totally empty.
- StorageComponent component(_node->getComponentRegister(), "test");
-
- metrics::MetricManager mm;
- MemoryStatusViewer viewer(
- *_memMan, mm, _node->getComponentRegister());
- std::ostringstream actual;
- viewer.reportStatus(actual, framework::HttpUrlPath("/"));
- CPPUNIT_ASSERT_MATCH_REGEX(".*Plotr.LineChart.*", actual.str());
- CPPUNIT_ASSERT_MATCH_REGEX(
- ".*Current: 1970-01-01 00:00:00 Max memory 1000 SnapShot\\(Used 0, w/o cache 0\\).*",
- actual.str());
- CPPUNIT_ASSERT_MATCH_REGEX(
- ".*Last hour: na.*", actual.str());
-}
-
-namespace {
- void waitForProcessedTime(
- const MemoryStatusViewer& viewer, framework::SecondTime time,
- framework::SecondTime timeout = framework::SecondTime(30))
- {
- framework::defaultimplementation::RealClock clock;
- framework::MilliSecTime endTime(
- clock.getTimeInMillis() + timeout.getMillis());
- framework::SecondTime processedTime(0);
- while (clock.getTimeInMillis() < endTime) {
- processedTime = viewer.getProcessedTime();
- if (processedTime >= time) return;
- FastOS_Thread::Sleep(1);
- }
- std::ostringstream ost;
- ost << "Timed out waiting " << timeout << " ms for time " << time
- << " to be processed. Currently time is only processed up to "
- << processedTime;
- throw new vespalib::IllegalStateException(ost.str(), VESPA_STRLOC);
- }
-}
-
-#define ASSERT_MEMORY(output, period, maxmem, used, usedwocache) \
-{ \
- std::string::size_type _pos1_(output.find(period)); \
- std::string::size_type _pos2_(output.find("Max memory", _pos1_)); \
- std::string::size_type _pos3_(output.find("SnapShot", _pos2_)); \
- std::string _maxMemory_(output.substr(_pos2_ + 11, _pos3_ - _pos2_ - 12)); \
- std::string::size_type _pos4_(output.find(",", _pos3_)); \
- std::string _used_(output.substr(_pos3_ + 14, _pos4_ - _pos3_ - 14)); \
- std::string::size_type _pos5_(output.find(")", _pos4_)); \
- std::string _usedwo_(output.substr(_pos4_ + 12, _pos5_ - _pos4_ - 12)); \
- std::ostringstream _failure_; \
- _failure_ << "Wrong match in period " << period << " in output:\n" \
- << output << "\nFor value: "; \
- \
- CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Max memory", \
- uint64_t(maxmem), boost::lexical_cast<uint64_t>(_maxMemory_)); \
- CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Used memory", \
- uint64_t(used), boost::lexical_cast<uint64_t>(_used_)); \
- CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Used memory w/o cache", \
- uint64_t(usedwocache), boost::lexical_cast<uint64_t>(_usedwo_)); \
-}
-
-void
-MemoryStatusViewerTest::testSnapshots()
-{
- // Add a memory manager, and add a bit of load to it, so it's not
- // totally empty.
- StorageComponent component(_node->getComponentRegister(), "test");
- const framework::MemoryAllocationType putAlloc(
- component.getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType("PUT")));
- const framework::MemoryAllocationType getAlloc(
- component.getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType("GET")));
-
- framework::MemoryToken::UP put = _memMan->allocate(putAlloc, 0, 100, 80);
- framework::MemoryToken::UP get = _memMan->allocate(getAlloc, 30, 200, 50);
- framework::MemoryToken::UP get2 = _memMan->allocate(getAlloc, 70, 150, 60);
-
- metrics::MetricManager mm;
- MemoryStatusViewer viewer(*_memMan, mm, _node->getComponentRegister());
-
- _node->getClock().addSecondsToTime(1000);
- viewer.notifyThread();
- waitForProcessedTime(viewer, framework::SecondTime(1000));
-
- std::ostringstream actual;
- viewer.printDebugOutput(actual);
- //std::cerr << actual.str() << "\n";
- ASSERT_MEMORY(actual.str(), "Current", 1000, 450, 450);
- ASSERT_MEMORY(actual.str(), "Last hour", 1000, 450, 450);
- ASSERT_MEMORY(actual.str(), "Last ever", 1000, 450, 450);
-
- put = _memMan->allocate(putAlloc, 0, 50, 80);
- get = _memMan->allocate(getAlloc, 100, 140, 50);
- get2 = _memMan->allocate(getAlloc, 20, 100, 70);
-
- _node->getClock().addSecondsToTime(3600);
- viewer.notifyThread();
- waitForProcessedTime(viewer, framework::SecondTime(4600));
-
- actual.str("");
- viewer.printDebugOutput(actual);
- //std::cerr << actual.str() << "\n";
- ASSERT_MEMORY(actual.str(), "Current", 1000, 290, 290);
- ASSERT_MEMORY(actual.str(), "Last hour", 1000, 540, 540);
- ASSERT_MEMORY(actual.str(), "Last ever", 1000, 540, 540);
-
- get.reset();
-
- _node->getClock().addSecondsToTime(3600);
- viewer.notifyThread();
- waitForProcessedTime(viewer, framework::SecondTime(4600 + 3600));
-
- actual.str("");
- viewer.printDebugOutput(actual);
- //std::cerr << actual.str() << "\n";
- ASSERT_MEMORY(actual.str(), "Current", 1000, 150, 150);
- ASSERT_MEMORY(actual.str(), "Last hour", 1000, 290, 290);
- ASSERT_MEMORY(actual.str(), "Last ever", 1000, 540, 540);
-
-}
-
-} // storage
diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
index 67b93108139..af0082fa788 100644
--- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
+++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
@@ -231,8 +231,6 @@ struct FileStorManagerTest : public CppUnit::TestFixture {
fprintf(stderr, "%s\n", e.what());
}
_testdoctype1 = _node->getTypeRepo()->getDocumentType("testdoctype1");
- _node->getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType("VISITOR_BUFFER"));
}
void putDoc(DummyStorageLink& top,
@@ -2010,19 +2008,11 @@ FileStorManagerTest::testVisiting()
top.reset();
// Visit bucket with no split, using no selection
{
- framework::MemoryToken::UP token(
- _node->getMemoryManager().allocate(
- _node->getMemoryManager().getAllocationType(
- "VISITOR_BUFFER"),
- 16*1024,
- 16*1024,
- 127));
spi::IteratorId iterId(createIterator(top, ids[0], "true"));
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), makeDocumentBucket(ids[0]), iterId, 16*1024));
+ auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[0]), iterId, 16*1024);
top.sendDown(cmd);
top.waitForMessages(1, _waitTime);
- CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies());
+ CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies());
std::shared_ptr<GetIterReply> reply(
std::dynamic_pointer_cast<GetIterReply>(top.getReply(0)));
CPPUNIT_ASSERT(reply.get());
@@ -2039,15 +2029,7 @@ FileStorManagerTest::testVisiting()
ids[1],
"testdoctype1.hstringval = \"John Doe\""));
while (true) {
- framework::MemoryToken::UP token(
- _node->getMemoryManager().allocate(
- _node->getMemoryManager().getAllocationType(
- "VISITOR_BUFFER"),
- 16*1024,
- 16*1024,
- 127));
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), makeDocumentBucket(ids[1]), iterId, 16*1024));
+ auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[1]), iterId, 16*1024);
top.sendDown(cmd);
top.waitForMessages(1, _waitTime);
CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies());
@@ -2078,18 +2060,10 @@ FileStorManagerTest::testVisiting()
true));
uint32_t totalDocs = 0;
while (true) {
- framework::MemoryToken::UP token(
- _node->getMemoryManager().allocate(
- _node->getMemoryManager().getAllocationType(
- "VISITOR_BUFFER"),
- 16*1024,
- 16*1024,
- 127));
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), makeDocumentBucket(ids[1]), iterId, 16*1024));
+ auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(ids[1]), iterId, 16*1024);
top.sendDown(cmd);
top.waitForMessages(1, _waitTime);
- CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies());
+ CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies());
std::shared_ptr<GetIterReply> reply(
std::dynamic_pointer_cast<GetIterReply>(
top.getReply(0)));
@@ -2799,15 +2773,8 @@ FileStorManagerTest::testGetIter()
// Sending a getiter request that will only visit some of the docs
spi::IteratorId iterId(createIterator(top, bid, ""));
{
- framework::MemoryToken::UP token(
- _node->getMemoryManager().allocate(
- _node->getMemoryManager().getAllocationType(
- "VISITOR_BUFFER"),
- 2048,
- 2048,
- 127));
std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), makeDocumentBucket(bid), iterId, 2048));
+ new GetIterCommand(makeDocumentBucket(bid), iterId, 2048));
top.sendDown(cmd);
top.waitForMessages(1, _waitTime);
CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies());
@@ -2837,18 +2804,10 @@ FileStorManagerTest::testGetIter()
CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult());
}
{
- framework::MemoryToken::UP token(
- _node->getMemoryManager().allocate(
- _node->getMemoryManager().getAllocationType(
- "VISITOR_BUFFER"),
- 2048,
- 2048,
- 127));
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), makeDocumentBucket(bid), iterId, 2048));
+ auto cmd = std::make_shared<GetIterCommand>(makeDocumentBucket(bid), iterId, 2048);
top.sendDown(cmd);
top.waitForMessages(1, _waitTime);
- CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies());
+ CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies());
std::shared_ptr<GetIterReply> reply(
std::dynamic_pointer_cast<GetIterReply>(
top.getReply(0)));
diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp
index fcfbcd0a78b..9d3a3d5f008 100644
--- a/storage/src/tests/persistence/persistencetestutils.cpp
+++ b/storage/src/tests/persistence/persistencetestutils.cpp
@@ -11,7 +11,6 @@
#include <vespa/vespalib/util/exceptions.h>
using document::DocumentType;
-using storage::framework::defaultimplementation::AllocationLogic;
using storage::spi::test::makeSpiBucket;
using document::test::makeDocumentBucket;
diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h
index ee87925a0bc..36b1fef7285 100644
--- a/storage/src/tests/persistence/persistencetestutils.h
+++ b/storage/src/tests/persistence/persistencetestutils.h
@@ -8,7 +8,6 @@
#include <vespa/storage/persistence/persistenceutil.h>
#include <vespa/storage/common/messagesender.h>
#include <vespa/storage/common/storagecomponent.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
#include <vespa/persistence/spi/persistenceprovider.h>
#include <vespa/persistence/dummyimpl/dummypersistence.h>
#include <vespa/document/base/testdocman.h>
diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp
index 5111368dd1c..cd5dd2a01a4 100644
--- a/storage/src/tests/storageserver/communicationmanagertest.cpp
+++ b/storage/src/tests/storageserver/communicationmanagertest.cpp
@@ -6,7 +6,6 @@
#include <vespa/messagebus/rpcmessagebus.h>
#include <vespa/storageapi/message/persistence.h>
#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h>
-#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h>
#include <tests/common/teststorageapp.h>
#include <tests/common/dummystoragelink.h>
#include <tests/common/testhelper.h>
diff --git a/storage/src/vespa/storage/CMakeLists.txt b/storage/src/vespa/storage/CMakeLists.txt
index 69846cafde2..d854c603097 100644
--- a/storage/src/vespa/storage/CMakeLists.txt
+++ b/storage/src/vespa/storage/CMakeLists.txt
@@ -12,7 +12,6 @@ vespa_add_library(storage
$<TARGET_OBJECTS:storage_bucketmover>
$<TARGET_OBJECTS:storage_thread>
$<TARGET_OBJECTS:storage_status>
- $<TARGET_OBJECTS:storage_memory>
$<TARGET_OBJECTS:storage_component>
INSTALL lib64
DEPENDS
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
index 677ca0a6309..aa9158f9e5b 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
@@ -34,7 +34,6 @@ BucketManager::BucketManager(const config::ConfigUri & configUri,
: StorageLinkQueued("Bucket manager", compReg),
framework::StatusReporter("bucketdb", "Bucket database"),
_configUri(configUri),
- _bucketDBMemoryToken(),
_workerLock(),
_workerCond(),
_clusterStateLock(),
@@ -48,12 +47,6 @@ BucketManager::BucketManager(const config::ConfigUri & configUri,
_requestsCurrentlyProcessing(0),
_component(compReg, "bucketmanager")
{
- const framework::MemoryAllocationType& allocType(
- _component.getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType("DATABASE")));
- _bucketDBMemoryToken = _component.getMemoryManager().allocate(
- allocType, 0, 0, api::StorageMessage::HIGH);
- assert(_bucketDBMemoryToken.get() != 0);
_metrics->setDisks(_component.getDiskCount());
_component.registerStatusPage(*this);
_component.registerMetric(*_metrics);
@@ -222,8 +215,6 @@ BucketManager::updateMetrics(bool updateDocCount)
LOG(debug, "Iterating bucket database to update metrics%s%s",
updateDocCount ? "" : ", minusedbits only",
_doneInitialized ? "" : ", server is not done initializing");
- uint64_t dbMemSize = _component.getBucketSpaceRepo().getBucketMemoryUsage();
- _bucketDBMemoryToken->resize(dbMemSize, dbMemSize);
uint32_t diskCount = _component.getDiskCount();
if (!updateDocCount || _doneInitialized) {
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h
index 5e4e928dfc7..8413c5a0f42 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.h
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h
@@ -20,7 +20,6 @@
#include <vespa/storage/common/servicelayercomponent.h>
#include <vespa/storage/common/storagelinkqueued.h>
#include <vespa/storageapi/message/bucket.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/generic/metric/metricupdatehook.h>
#include <vespa/storageframework/generic/status/statusreporter.h>
@@ -47,7 +46,6 @@ private:
config::ConfigUri _configUri;
uint32_t _chunkLevel;
- framework::MemoryToken::UP _bucketDBMemoryToken;
BucketInfoRequestMap _bucketInfoRequests;
/**
diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp
index de44116b316..d458a821e02 100644
--- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp
+++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "statusmetricconsumer.h"
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/generic/status/htmlstatusreporter.h>
#include <boost/assign.hpp>
#include <boost/lexical_cast.hpp>
@@ -26,17 +25,8 @@ StatusMetricConsumer::StatusMetricConsumer(
_component(compReg, "statusmetricsconsumer"),
_name(name),
_startTime(_component.getClock().getTimeInSeconds()),
- _processedTime(0),
- _metricMemoryToken()
+ _processedTime(0)
{
- const framework::MemoryAllocationType& allocType(
- _component.getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType(
- "METRICS", framework::MemoryAllocationType::FORCE_ALLOCATE)
- ));
- _metricMemoryToken = _component.getMemoryManager().allocate(
- allocType, 0, 0, api::StorageMessage::HIGH);
- assert(_metricMemoryToken.get() != 0);
LOG(debug, "Started metrics consumer");
setlocale(LC_NUMERIC, "");
_component.registerMetricUpdateHook(*this, framework::SecondTime(3600));
@@ -51,8 +41,7 @@ void
StatusMetricConsumer::updateMetrics(const MetricLockGuard & guard)
{
metrics::MemoryConsumption::UP mc(_manager.getMemoryConsumption(guard));
- uint32_t usage = mc->getTotalMemoryUsage();
- _metricMemoryToken->resize(usage, usage);
+ // TODO is this hook needed anymore?
}
vespalib::string
diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.h b/storage/src/vespa/storage/common/statusmetricconsumer.h
index e3bae51cf99..6f93f51cfdf 100644
--- a/storage/src/vespa/storage/common/statusmetricconsumer.h
+++ b/storage/src/vespa/storage/common/statusmetricconsumer.h
@@ -53,7 +53,6 @@ private:
vespalib::Monitor _waiter;
framework::SecondTime _startTime;
framework::SecondTime _processedTime;
- std::unique_ptr<framework::MemoryToken> _metricMemoryToken;
void writeXmlTags(std::ostream& out,
const vespalib::StringTokenizer& name,
diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h
index a3c9804c2b4..29e8d3f6221 100644
--- a/storage/src/vespa/storage/distributor/bucketdbupdater.h
+++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h
@@ -13,7 +13,6 @@
#include <vespa/vdslib/state/clusterstate.h>
#include <vespa/storage/common/storagelink.h>
#include <vespa/storageframework/generic/clock/timer.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageapi/messageapi/messagehandler.h>
#include <set>
#include <deque>
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h
index e8480902549..8d31a228543 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h
@@ -6,7 +6,6 @@
#include <vespa/storageapi/messageapi/storagereply.h>
#include <vespa/storageapi/messageapi/maintenancecommand.h>
#include <vespa/document/bucket/bucketid.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
namespace storage::distributor {
@@ -229,7 +228,6 @@ protected:
bool _ok;
api::StorageMessage::Priority _priority;
- framework::MemoryToken::UP _memoryToken;
/**
* Checks if the given bucket is blocked by any pending messages to any
diff --git a/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt b/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt
deleted file mode 100644
index 2a91ead2b60..00000000000
--- a/storage/src/vespa/storage/frameworkimpl/memory/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(storage_memory OBJECT
- SOURCES
- memorystatusviewer.cpp
- DEPENDS
- AFTER
- storage_storageconfig
-)
diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h b/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h
deleted file mode 100644
index fd47bec1424..00000000000
--- a/storage/src/vespa/storage/frameworkimpl/memory/memorysnapshotlist.h
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::MemorySnapshotList
- *
- * \brief Holds a historic list of MemoryStates.
- *
- */
-
-#pragma once
-
-#include <storage/frameworkimpl/memory/memorystate.h>
-
-namespace storage {
-
-class MemorySnapshotList : public vespalib::Printable {
- std::map<uint64_t time, MemoryState> _snapshots;
-};
-
-} // storage
-
diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp
deleted file mode 100644
index dd07a61f27b..00000000000
--- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp
+++ /dev/null
@@ -1,661 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memorystatusviewer.h"
-#include <vespa/storage/storageutil/graph.h>
-#include <vespa/storage/storageutil/palette.h>
-#include <vespa/storage/storageutil/piechart.h>
-#include <vespa/metrics/metricmanager.h>
-#include <vespa/storageapi/messageapi/storagemessage.h>
-#include <algorithm>
-
-#include <vespa/log/bufferedlogger.h>
-LOG_SETUP(".memory.status.viewer");
-
-using storage::framework::defaultimplementation::MemoryState;
-
-namespace storage {
-
-MemoryStatusViewer::Entry::Entry(
- const std::string& name, framework::Clock& clock,
- framework::SecondTime maxAge)
- : _name(name),
- _maxAge(maxAge),
- _timeTaken(clock.getTimeInSeconds()),
- _data(),
- _maxMemory(0)
-{
-}
-
-MemoryStatusViewer::MemoryStatusViewer(
- framework::defaultimplementation::MemoryManager& mm,
- const metrics::MetricManager& metricMan,
- StorageComponentRegister& compReg)
- : framework::HtmlStatusReporter("memorymanager", "Memory Manager"),
- _component(compReg, "memorystatusviewer"),
- _manager(mm),
- _metricManager(metricMan),
- _workerMonitor(),
- _states(),
- _memoryHistory(),
- _memoryHistorySize(24 * 31),
- _memoryHistoryPeriod(60),
- _allowedSlackPeriod(6),
- _lastHistoryUpdate(_component.getClock().getTimeInSeconds()),
- _processedTime(0)
-{
- addEntry("Current", 0);
- addEntry("Last hour", 60 * 60);
- addEntry("Last day", 24 * 60 * 60);
- addEntry("Last month", 4 * 7 * 24 * 60 * 60);
- addEntry("Last ever", std::numeric_limits<uint32_t>::max());
-
- framework::MilliSecTime maxProcessingTime(60 * 1000);
- _thread = _component.startThread(*this, maxProcessingTime,
- framework::MilliSecTime(1000));
- _component.registerStatusPage(*this);
-}
-
-MemoryStatusViewer::~MemoryStatusViewer()
-{
- if (_thread.get() != 0) {
- _thread->interrupt();
- {
- vespalib::MonitorGuard monitor(_workerMonitor);
- monitor.signal();
- }
- _thread->join();
- }
-}
-
-namespace {
- struct Group {
- std::set<const framework::MemoryAllocationType*> types;
- api::StorageMessage::Priority minPri;
- api::StorageMessage::Priority maxPri;
- MemoryState::Entry entry;
-
- Group(const framework::MemoryAllocationType& type,
- api::StorageMessage::Priority pri,
- const MemoryState::Entry& e)
- : types(),
- minPri(pri),
- maxPri(pri),
- entry(e)
- {
- types.insert(&type);
- }
- };
-
- struct GroupSizeOrder {
- bool operator()(const Group& g1, const Group& g2) {
- return (g1.entry._currentUsedSize > g2.entry._currentUsedSize);
- }
- };
- struct GroupAllocsOrder {
- bool operator()(const Group& g1, const Group& g2) {
- return (g1.entry._totalUserCount > g2.entry._totalUserCount);
- }
- };
- struct GroupMinAllocsOrder {
- bool operator()(const Group& g1, const Group& g2) {
- return (g1.entry._minimumCount > g2.entry._minimumCount);
- }
- };
- struct GroupDeniedAllocsOrder {
- bool operator()(const Group& g1, const Group& g2) {
- return (g1.entry._deniedCount > g2.entry._deniedCount);
- }
- };
-
- std::vector<Group> collapsePriorities(MemoryStatusViewer::Entry& entry) {
- std::vector<Group> groups;
- const MemoryState::SnapShot& ss(entry._data);
- for (MemoryState::AllocationMap::const_iterator it
- = ss.getAllocations().begin();
- it != ss.getAllocations().end(); ++it)
- {
- std::unique_ptr<Group> group;
- for (MemoryState::PriorityMap::const_iterator it2
- = it->second.begin(); it2 != it->second.end(); ++it2)
- {
- if (group.get() == 0) {
- group.reset(new Group(
- *it->first, it2->first, it2->second));
- } else {
- group->entry += it2->second;
- group->minPri = std::min(group->minPri, it2->first);
- group->maxPri = std::max(group->maxPri, it2->first);
- }
- }
- if (group.get() != 0) {
- groups.push_back(*group);
- }
- }
- return groups;
- }
-
- std::vector<Group> groupLoad(uint32_t groupCount, uint64_t minSize,
- uint32_t minEntries,
- MemoryStatusViewer::Entry& entry)
- {
- assert(groupCount > 1);
- std::vector<Group> groups(collapsePriorities(entry));
- if (groups.size() == 0) return groups;
- std::sort(groups.begin(), groups.end(), GroupSizeOrder());
- assert(groups.front().entry._currentUsedSize
- >= groups.back().entry._currentUsedSize);
- while (groups.size() > minEntries
- && (groups.size() > groupCount
- || groups[groups.size() - 2].entry._currentUsedSize
- < minSize))
- {
- Group& nextButLast(groups[groups.size() - 2]);
- Group& last(groups.back());
- if (last.entry._currentUsedSize > 0) {
- nextButLast.entry += last.entry;
- nextButLast.minPri = std::min(nextButLast.minPri, last.minPri);
- nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri);
- nextButLast.types.insert(*last.types.begin());
- }
- groups.pop_back();
- }
- return groups;
- }
-
- std::vector<Group> groupAllocs(uint32_t groupCount, uint64_t minSize,
- uint32_t minEntries,
- MemoryStatusViewer::Entry& entry)
- {
- assert(groupCount > 1);
- std::vector<Group> groups(collapsePriorities(entry));
- if (groups.size() == 0) return groups;
- std::sort(groups.begin(), groups.end(), GroupAllocsOrder());
- assert(groups.front().entry._totalUserCount
- >= groups.back().entry._totalUserCount);
- while (groups.size() > minEntries
- && (groups.size() > groupCount
- || groups[groups.size() - 2].entry._totalUserCount
- < minSize))
- {
- Group& nextButLast(groups[groups.size() - 2]);
- Group& last(groups.back());
- nextButLast.entry += last.entry;
- nextButLast.minPri = std::min(nextButLast.minPri, last.minPri);
- nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri);
- nextButLast.types.insert(*last.types.begin());
- groups.pop_back();
- }
- return groups;
- }
-
- std::vector<Group> groupMinAllocs(uint32_t groupCount, uint64_t minSize,
- uint32_t minEntries,
- MemoryStatusViewer::Entry& entry)
- {
- assert(groupCount > 1);
- std::vector<Group> groups(collapsePriorities(entry));
- if (groups.size() == 0) return groups;
- std::sort(groups.begin(), groups.end(), GroupMinAllocsOrder());
- assert(groups.front().entry._minimumCount
- >= groups.back().entry._minimumCount);
- while (groups.size() > minEntries
- && (groups.size() > groupCount
- || groups[groups.size() - 2].entry._minimumCount
- < minSize))
- {
- Group& nextButLast(groups[groups.size() - 2]);
- Group& last(groups.back());
- nextButLast.entry += last.entry;
- nextButLast.minPri = std::min(nextButLast.minPri, last.minPri);
- nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri);
- nextButLast.types.insert(*last.types.begin());
- groups.pop_back();
- }
- return groups;
- }
-
- std::vector<Group> groupDeniedAllocs(uint32_t groupCount, uint64_t minSize,
- uint32_t minEntries,
- MemoryStatusViewer::Entry& entry)
- {
- assert(groupCount > 1);
- std::vector<Group> groups(collapsePriorities(entry));
- if (groups.size() == 0) return groups;
- std::sort(groups.begin(), groups.end(), GroupDeniedAllocsOrder());
- assert(groups.front().entry._deniedCount
- >= groups.back().entry._deniedCount);
- while (groups.size() > minEntries
- && (groups.size() > groupCount
- || groups[groups.size() - 2].entry._deniedCount
- < minSize))
- {
- Group& nextButLast(groups[groups.size() - 2]);
- Group& last(groups.back());
- nextButLast.entry += last.entry;
- nextButLast.minPri = std::min(nextButLast.minPri, last.minPri);
- nextButLast.maxPri = std::max(nextButLast.maxPri, last.maxPri);
- nextButLast.types.insert(*last.types.begin());
- groups.pop_back();
- }
- return groups;
- }
-}
-
-void
-MemoryStatusViewer::printSnapshot(
- std::ostream& out, Entry& entry,
- std::map<const framework::MemoryAllocationType*,
- uint32_t>& colors) const
-{
- out << "<h4>" << entry._name << " - Taken at "
- << entry._timeTaken.toString() << "</h4>\n"
- << "<table><tr><td>\n"
- << "<b>Memory usage";
- if (entry._name != "Current") {
- out << ", maxed at " << framework::SecondTime(entry._timeTaken);
- }
- out << " with "
- << (entry._data.getUsedSizeIgnoringCache() / (1024 * 1024))
- << " MB.</b><br>\n";
- std::string piename = entry._name;
- std::replace(piename.begin(), piename.end(), ' ', '_');
- uint64_t freeSize = entry._maxMemory - entry._data.getUsedSize();
- // Memory usage pie
- uint64_t minSize = freeSize / 20;
- std::vector<Group> groups(groupLoad(20, minSize, 5, entry));
- PieChart chart(piename, PieChart::SCHEME_CUSTOM);
- chart.printLabels(false);
- for (uint32_t i=0; i<groups.size(); ++i) {
- std::string name = "Other";
- if (groups[i].types.size() == 1) {
- name = (*groups[i].types.begin())->getName();
- }
- uint32_t mbytes = groups[i].entry._currentUsedSize / (1024 * 1024);
- std::ostringstream ost;
- ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri);
- if (groups[i].minPri != groups[i].maxPri) {
- ost << " - " << static_cast<uint16_t>(groups[i].maxPri);
- }
- ost << " (" << mbytes << " MB)";
- name = ost.str();
- if (groups[i].entry._currentUsedSize > 0) {
- chart.add(groups[i].entry._currentUsedSize, name,
- colors[*groups[i].types.begin()]);
- }
- }
- {
- std::ostringstream ost;
- ost << "Free (" << (freeSize / (1024 * 1024)) << " MB)";
- chart.add(freeSize, ost.str(), colors[0]);
- }
- chart.printCanvas(out, 750, 300);
- out << "\n\n";
- chart.printScript(out, "");
- out << "\n\n";
- // Total allocations pie
- out << "</td><td>\n";
- PieChart allocChart(piename + "Alloc", PieChart::SCHEME_CUSTOM);
- allocChart.printLabels(false);
- groups = groupAllocs(20, 100, 5, entry);
- uint64_t totalAllocs = 0;
- for (uint32_t i=0; i<groups.size(); ++i) {
- std::string name = "Other";
- if (groups[i].types.size() == 1) {
- name = (*groups[i].types.begin())->getName();
- }
- uint32_t allocs = groups[i].entry._totalUserCount;
- totalAllocs += allocs;
- std::ostringstream ost;
- ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri);
- if (groups[i].minPri != groups[i].maxPri) {
- ost << " - " << static_cast<uint16_t>(groups[i].maxPri);
- }
- ost << " (" << allocs << " allocations)";
- name = ost.str();
- if (groups[i].entry._totalUserCount > 0) {
- allocChart.add(groups[i].entry._totalUserCount, name,
- colors[*groups[i].types.begin()]);
- }
- }
- out << "<b>Allocations, totalling " << totalAllocs << "</b><br>\n";
- allocChart.printCanvas(out, 750, 300);
- out << "\n\n";
- allocChart.printScript(out, "");
- out << "\n\n";
- out << "</td></tr><tr><td>\n";
- PieChart minChart(piename + "Min", PieChart::SCHEME_CUSTOM);
- minChart.printLabels(false);
- groups = groupMinAllocs(20, 100, 5, entry);
- uint64_t totalMinAllocs = 0;
- for (uint32_t i=0; i<groups.size(); ++i) {
- std::string name = "Other";
- if (groups[i].types.size() == 1) {
- name = (*groups[i].types.begin())->getName();
- }
- uint32_t allocs = groups[i].entry._minimumCount;
- totalMinAllocs += allocs;
- std::ostringstream ost;
- ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri);
- if (groups[i].minPri != groups[i].maxPri) {
- ost << " - " << static_cast<uint16_t>(groups[i].maxPri);
- }
- ost << " (" << allocs << " min allocations)";
- name = ost.str();
- if (groups[i].entry._minimumCount > 0) {
- minChart.add(groups[i].entry._minimumCount, name,
- colors[*groups[i].types.begin()]);
- }
- }
- out << "<b>Minimum allocations, totalling " << totalMinAllocs
- << "</b><br>\n";
- if (totalMinAllocs > 0) {
- minChart.printCanvas(out, 750, 300);
- out << "\n\n";
- minChart.printScript(out, "");
- out << "\n\n";
- }
- out << "</td><td>\n";
- PieChart deniedChart(piename + "Denied", PieChart::SCHEME_CUSTOM);
- deniedChart.printLabels(false);
- groups = groupDeniedAllocs(20, 100, 5, entry);
- uint64_t totalDeniedAllocs = 0;
- for (uint32_t i=0; i<groups.size(); ++i) {
- std::string name = "Other";
- if (groups[i].types.size() == 1) {
- name = (*groups[i].types.begin())->getName();
- }
- uint32_t allocs = groups[i].entry._deniedCount;
- totalDeniedAllocs += allocs;
- std::ostringstream ost;
- ost << name << ", pri " << static_cast<uint16_t>(groups[i].minPri);
- if (groups[i].minPri != groups[i].maxPri) {
- ost << " - " << static_cast<uint16_t>(groups[i].maxPri);
- }
- ost << " (" << allocs << " denied allocations)";
- name = ost.str();
- if (groups[i].entry._deniedCount > 0) {
- deniedChart.add(groups[i].entry._deniedCount, name,
- colors[*groups[i].types.begin()]);
- }
- }
- out << "<b>Denied allocations, totalling " << totalDeniedAllocs
- << "</b><br>\n";
- if (totalDeniedAllocs > 0) {
- deniedChart.printCanvas(out, 750, 300);
- out << "\n\n";
- deniedChart.printScript(out, "");
- out << "\n\n";
- }
- out << "</td></tr></table>\n";
-}
-
-void
-MemoryStatusViewer::reportHtmlHeaderAdditions(
- std::ostream& out, const framework::HttpUrlPath&) const
-{
- (void) out;
- // FIXME this function used to emit Yahoo-internal links to graph plotting
- // JS files. Obviously, this won't work for external users. Either way, the
- // memory manager/status reporter is deprecated.
-}
-
-namespace {
- std::map<const framework::MemoryAllocationType*, uint32_t> assignColors(
- const std::vector<const framework::MemoryAllocationType*>& types)
- {
- Palette palette(types.size() + 1);
- std::map<const framework::MemoryAllocationType*, uint32_t> colors;
- uint32_t nextCol = 0;
- colors[0] = palette[nextCol++];
- for (std::vector<const framework::MemoryAllocationType*>
- ::const_iterator it = types.begin(); it != types.end(); ++it)
- {
- colors[*it] = palette[nextCol++];
- }
- return colors;
- }
-}
-
-void
-MemoryStatusViewer::reportHtmlStatus(std::ostream& out,
- const framework::HttpUrlPath& path) const
-{
- vespalib::MonitorGuard monitor(_workerMonitor);
-
- if (path.getAttribute("page") == "reset") {
- }
- if (path.getAttribute("interval") == "current") {
- Entry& e(*_states[0]);
- out << "<pre>" << e._name << ": ";
- if (e.containsData()) {
- e._data.print(out, true, " ");
- } else {
- out << "na";
- }
- out << "\n</pre>";
- return;
- }
- const_cast<MemoryStatusViewer*>(this)->grabMemoryUsage();
- framework::SecondTime currentTime(_component.getClock().getTimeInSeconds());
- std::vector<const framework::MemoryAllocationType*> allocTypes(
- _manager.getAllocationTypes());
- std::map<const framework::MemoryAllocationType*, uint32_t> colors(
- assignColors(allocTypes));
- // Print memory usage graph
- {
- uint32_t mb = 1024 * 1024;
- Graph memoryHistory("memhistory", Graph::SCHEME_CUSTOM);
- std::vector<Graph::Point> total;
- std::vector<Graph::Point> used;
- std::vector<Graph::Point> usedWoCache;
- uint32_t xval = 0;
- for (std::deque<MemoryTimeEntry>::const_iterator it
- = _memoryHistory.begin(); it != _memoryHistory.end();
- ++it, ++xval)
- {
- used.push_back(Graph::Point(xval, it->used));
- usedWoCache.push_back(Graph::Point(xval, it->usedWithoutCache));
- }
- used.push_back(Graph::Point(
- xval, _states[0]->_data.getUsedSize() / mb));
- usedWoCache.push_back(Graph::Point(
- xval, _states[0]->_data.getUsedSizeIgnoringCache() / mb));
- uint32_t totalSize = _states[0]->_maxMemory / mb;
- total.push_back(Graph::Point(0, totalSize));
- total.push_back(Graph::Point(xval, totalSize));
- memoryHistory.add(total, "Total memory", Graph::GREEN);
- memoryHistory.add(used, "Used memory", Graph::YELLOW);
- memoryHistory.add(usedWoCache, "Used memory excluding freeable cache",
- Graph::RED);
- out << "<p>Memory available for lowest priority (255): "
- << _manager.getMemorySizeFreeForPriority(255) << " byte(s).</p>\n";
- out << "<h3>Historic memory usage</h3>\n";
- uint32_t yAxisUnit = ((totalSize / 4) / 256) * 256;
- if (yAxisUnit == 0) yAxisUnit = (totalSize / 4);
- if (yAxisUnit == 0) yAxisUnit = 1;
- uint32_t size = yAxisUnit;
- memoryHistory.addYAxisLabel(0, "0 B");
- while (size <= totalSize) {
- std::ostringstream label;
- if (size % 1024 == 0) {
- label << (size / 1024) << " GB";
- } else {
- label << size << " MB";
- }
- memoryHistory.addYAxisLabel(size, label.str());
- size += yAxisUnit;
- }
- uint32_t xAxisUnit = ((_memoryHistory.size() / 4) / 24) * 24;
- if (xAxisUnit == 0) xAxisUnit = _memoryHistoryPeriod.getTime();
- uint32_t startTime = ((currentTime.getTime()
- / _memoryHistoryPeriod.getTime())
- / 24) * 24;
- uint32_t stopTime = (currentTime.getTime()
- / _memoryHistoryPeriod.getTime())
- - _memoryHistory.size() + 1;
- memoryHistory.addXAxisLabel(xval, currentTime.toString());
- bool addedMiddlePoints = false;
- while (startTime >= stopTime) {
- if (currentTime.getTime() / _memoryHistoryPeriod.getTime()
- - startTime > 48)
- {
- memoryHistory.addXAxisLabel(
- (startTime - stopTime),
- framework::SecondTime(
- startTime * _memoryHistoryPeriod.getTime())
- .toString());
- addedMiddlePoints = true;
- }
- startTime -= xAxisUnit;
- }
- if (!addedMiddlePoints && _memoryHistory.size() > 2) {
- memoryHistory.addXAxisLabel(
- 1,
- framework::SecondTime(
- stopTime * _memoryHistoryPeriod.getTime())
- .toString());
- }
- memoryHistory.setBorders(50, 0, 0, 30);
- memoryHistory.setLegendPos(80, 20);
- memoryHistory.printCanvas(out, 1000, 250);
- memoryHistory.printScript(out, "");
- }
- uint32_t maxUsedWithoutCache = 0;
- for (uint32_t i=0; i<_states.size(); ++i) {
- Entry& e(*_states[i]);
- if (!e.containsData()
- || e._data.getUsedSizeIgnoringCache() == maxUsedWithoutCache)
- {
- continue;
- }
- printSnapshot(out, e, colors);
- maxUsedWithoutCache = e._data.getUsedSizeIgnoringCache();
- }
- out << "<h3>Raw output of stored data</h3>\n"
- << "<pre>\n";
- monitor.unlock();
- printDebugOutput(out);
- out << "</pre>\n";
- out << "<h2>Memory used for metrics. (Not tracked in memory manager)</h2>\n"
- << "<pre>\n"
- << _metricManager.getMemoryConsumption(_metricManager.getMetricLock())->toString()
- << "\n</pre>\n";
-}
-
-void
-MemoryStatusViewer::run(framework::ThreadHandle& thread)
-{
- while (!thread.interrupted()) {
- vespalib::MonitorGuard monitor(_workerMonitor);
- framework::SecondTime currentTime(
- _component.getClock().getTimeInSeconds());
- if (_lastHistoryUpdate + _memoryHistoryPeriod <= currentTime
- || _states[0]->_timeTaken + _memoryHistoryPeriod <= currentTime)
- {
- grabMemoryUsage();
- _processedTime = currentTime;
- LOG(spam, "Done processing time %" PRIu64, currentTime.getTime());
- thread.registerTick(framework::PROCESS_CYCLE);
- } else {
- monitor.wait(thread.getWaitTime());
- thread.registerTick(framework::WAIT_CYCLE);
- }
- }
-}
-
-// You should have worker monitor when calling this function
-void
-MemoryStatusViewer::grabMemoryUsage()
-{
- framework::SecondTime currentTime(_component.getClock().getTimeInSeconds());
- MemoryState state(_component.getClock(), 0);
- _manager.getState(state, true);
-
- if (_lastHistoryUpdate + _memoryHistoryPeriod <= currentTime) {
- LOG(spam, "Adding another %" PRIu64 " sec entry to memory history.",
- _memoryHistoryPeriod.getTime());
- // Add history once an hour
- uint32_t mb = 1024 * 1024;
- _memoryHistory.push_back(MemoryTimeEntry(
- state.getMaxSnapshot().getUsedSize() / mb,
- state.getMaxSnapshot().getUsedSizeIgnoringCache() / mb));
- if (_memoryHistory.size() > _memoryHistorySize) {
- if (_memoryHistoryPeriod != framework::SecondTime(60 * 60)) {
- uint32_t periodDiff = 60 * 60 / _memoryHistoryPeriod.getTime();
- std::deque<MemoryTimeEntry> newHistory;
- uint32_t count = 0;
- MemoryTimeEntry entry(0, 0);
- for (std::deque<MemoryTimeEntry>::const_iterator it
- = _memoryHistory.begin();
- it != _memoryHistory.end(); ++it)
- {
- entry.keepMax(*it);
- if (++count == periodDiff) {
- newHistory.push_back(entry);
- entry = MemoryTimeEntry(0, 0);
- count = 0;
- }
- }
- if (entry.used != 0) {
- newHistory.push_back(entry);
- }
- _memoryHistory.swap(newHistory);
- _memoryHistoryPeriod = framework::SecondTime(60 * 60);
- }
- }
- _lastHistoryUpdate += _memoryHistoryPeriod;
- if (_lastHistoryUpdate + _allowedSlackPeriod < currentTime) {
- LOGBP(warning, "Memory history is supposed to be tracked every %"
- PRIu64 " seconds, but %" PRIu64" seconds have passed "
- "since last update. Memory history graph will be "
- "incorrect.",
- _memoryHistoryPeriod.getTime(),
- (currentTime - _lastHistoryUpdate + _memoryHistoryPeriod)
- .getTime());
- _lastHistoryUpdate = currentTime;
- }
- }
- LOG(spam, "Overwriting current with snapshot using %" PRIu64 " bytes.",
- state.getCurrentSnapshot().getUsedSize());
- _states[0]->assign(state.getCurrentSnapshot(),
- state.getTotalSize(), currentTime);
- for (uint32_t i=1, n=_states.size(); i<n; ++i) {
- if (currentTime - _states[i]->_timeTaken >= _states[i]->_maxAge
- || state.getMaxSnapshot().getUsedSize()
- > _states[i]->_data.getUsedSize())
- {
- LOG(spam, "Updating period %s usage. Old usage was %" PRIu64 ". "
- "Last set at %" PRIu64,
- _states[i]->_name.c_str(), _states[i]->_data.getUsedSize(),
- _states[i]->_timeTaken.getTime());
- _states[i]->assign(state.getMaxSnapshot(),
- state.getTotalSize(), currentTime);
- }
- }
-}
-
-void
-MemoryStatusViewer::notifyThread() const
-{
- vespalib::MonitorGuard monitor(_workerMonitor);
- monitor.broadcast();
-}
-
-void
-MemoryStatusViewer::printDebugOutput(std::ostream& out) const
-{
- vespalib::MonitorGuard monitor(_workerMonitor);
- for (uint32_t i=0; i<_states.size(); ++i) {
- Entry& e(*_states[i]);
- out << e._name << ": ";
- if (e.containsData()) {
- out << e._timeTaken.toString() << " Max memory " << e._maxMemory << " ";
- e._data.print(out, true, " ");
- } else {
- out << "na";
- }
- out << "\n\n";
- }
-}
-
-} // storage
diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h
deleted file mode 100644
index 2c7510d371d..00000000000
--- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h
+++ /dev/null
@@ -1,137 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::MemoryStatusViewer
- *
- * \brief Generates status to access through status pages.
- *
- * Keeps a history of the largest memory inprints seen historically. This is
- * done be defining periods, where a period is always a multiplum of the length
- * of the period shorter than it. The last entry will store the biggest memory
- * imprint ever seen, and the earlier entries will show biggest for their time
- * period.
- *
- * To avoid having all periods cleared once the biggest period resets, the
- * periods keep data for each of the periods one size below it. Thus, a year
- * keeps data for 12 months, a month for 30 days, and so on.
- *
- * The memory state objects are divided in 3 parts. Current memory data, max
- * memory data since since reset and counts for how often various events have
- * happened.
- *
- * The counts will have their total count values stored in the current entry.
- * When the next period is updated getting a copy of these counts, we can see
- * how many counts have happened recently, by taking the current entry and
- * subtract those accounted for earlier.
- *
- * The current memory data will not be interesting for anything than to show the
- * actual now values in the current entry.
- *
- * The max since reset values will be the values used for the various periods.
- * When a period is updated with new data for a subpart of their period, the
- * max seen data is reset in the period in front, such that a lower maximum
- * can be found.
- */
-
-#pragma once
-
-#include <vespa/storage/common/storagecomponent.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorystate.h>
-#include <vespa/storageframework/generic/status/htmlstatusreporter.h>
-#include <vespa/vespalib/util/document_runnable.h>
-#include <vespa/vespalib/util/sync.h>
-#include <deque>
-#include <vector>
-
-
-namespace metrics {
- class MetricManager;
-}
-
-namespace storage {
-
-class StorageServerInterface;
-
-class MemoryStatusViewer : public framework::HtmlStatusReporter,
- private framework::Runnable
-{
-public:
- typedef framework::defaultimplementation::MemoryState::SnapShot SnapShot;
- struct Entry {
- typedef std::shared_ptr<Entry> SP;
-
- std::string _name;
- framework::SecondTime _maxAge;
- framework::SecondTime _timeTaken;
- SnapShot _data;
- uint64_t _maxMemory;
-
- Entry(const std::string& name, framework::Clock&,
- framework::SecondTime maxAge);
- bool containsData() const { return (_maxMemory != 0); }
-
- void assign(const SnapShot& snapshot, uint64_t maxMemory,
- framework::SecondTime time)
- {
- _data = snapshot;
- _maxMemory = maxMemory;
- _timeTaken = time;
- }
- };
-
- struct MemoryTimeEntry {
- uint64_t used;
- uint64_t usedWithoutCache;
-
- MemoryTimeEntry(uint64_t u, uint64_t wo)
- : used(u), usedWithoutCache(wo) {}
-
- void keepMax(const MemoryTimeEntry& e) {
- used = (used > e.used ? used : e.used);
- usedWithoutCache = (usedWithoutCache > e.usedWithoutCache
- ? usedWithoutCache : e.usedWithoutCache);
- }
- };
-
-private:
- framework::Component _component;
- framework::defaultimplementation::MemoryManager& _manager;
- const metrics::MetricManager& _metricManager;
- vespalib::Monitor _workerMonitor;
-
- std::vector<Entry::SP> _states;
- std::deque<MemoryTimeEntry> _memoryHistory;
- uint32_t _memoryHistorySize;
- framework::SecondTime _memoryHistoryPeriod;
- framework::SecondTime _allowedSlackPeriod;
- framework::SecondTime _lastHistoryUpdate;
- framework::Thread::UP _thread;
- framework::SecondTime _processedTime;
-
- void addEntry(const std::string& name, uint32_t maxAge) {
- _states.push_back(Entry::SP(new Entry(name, _component.getClock(),
- framework::SecondTime(maxAge))));
- }
- void run(framework::ThreadHandle&) override;
- void grabMemoryUsage();
- void printSnapshot(std::ostream& out, Entry& entry,
- std::map<const framework::MemoryAllocationType*,
- uint32_t>& colors) const;
-
-public:
- MemoryStatusViewer(
- framework::defaultimplementation::MemoryManager&,
- const metrics::MetricManager&,
- StorageComponentRegister&);
- ~MemoryStatusViewer();
-
- void reportHtmlHeaderAdditions(std::ostream&, const framework::HttpUrlPath&) const override;
- void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override;
-
- /** Useful for testing. */
- framework::SecondTime getProcessedTime() const { return _processedTime; }
- void notifyThread() const;
- void printDebugOutput(std::ostream&) const;
-
-};
-
-}
diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp
index 876067e4e15..720371a68f1 100644
--- a/storage/src/vespa/storage/persistence/messages.cpp
+++ b/storage/src/vespa/storage/persistence/messages.cpp
@@ -6,17 +6,14 @@ using document::BucketSpace;
namespace storage {
-GetIterCommand::GetIterCommand(framework::MemoryToken::UP token,
- const document::Bucket &bucket,
+GetIterCommand::GetIterCommand(const document::Bucket &bucket,
const spi::IteratorId iteratorId,
uint32_t maxByteSize)
: api::InternalCommand(ID),
- _token(std::move(token)),
_bucket(bucket),
_iteratorId(iteratorId),
_maxByteSize(maxByteSize)
{
- assert(_token.get());
}
GetIterCommand::~GetIterCommand() { }
@@ -38,7 +35,6 @@ GetIterCommand::makeReply() {
GetIterReply::GetIterReply(GetIterCommand& cmd)
: api::InternalReply(ID, cmd),
- _token(cmd.releaseMemoryToken()),
_bucket(cmd.getBucket()),
_completed(false)
{ }
diff --git a/storage/src/vespa/storage/persistence/messages.h b/storage/src/vespa/storage/persistence/messages.h
index 7bbeea8a12a..ba7f5979569 100644
--- a/storage/src/vespa/storage/persistence/messages.h
+++ b/storage/src/vespa/storage/persistence/messages.h
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
-#include <vespa/storageframework/generic/memory/memorytoken.h>
#include <vespa/storageapi/message/internal.h>
#include <vespa/persistence/spi/docentry.h>
#include <vespa/persistence/spi/bucket.h>
@@ -14,7 +13,6 @@ namespace storage {
class GetIterCommand : public api::InternalCommand {
private:
- mutable framework::MemoryToken::UP _token;
document::Bucket _bucket;
spi::IteratorId _iteratorId;
uint32_t _maxByteSize;
@@ -24,8 +22,7 @@ public:
typedef std::unique_ptr<GetIterCommand> UP;
typedef std::shared_ptr<GetIterCommand> SP;
- GetIterCommand(framework::MemoryToken::UP token,
- const document::Bucket &bucket,
+ GetIterCommand(const document::Bucket &bucket,
const spi::IteratorId iteratorId,
uint32_t maxByteSize);
~GetIterCommand();
@@ -44,13 +41,11 @@ public:
void print(std::ostream& out, bool verbose, const std::string& indent) const override;
private:
- framework::MemoryToken::UP releaseMemoryToken() { return std::move(_token); }
friend class GetIterReply;
};
class GetIterReply : public api::InternalReply {
private:
- framework::MemoryToken::UP _token;
document::Bucket _bucket;
std::vector<spi::DocEntry::UP> _entries;
bool _completed;
diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt
index 3d0a0fdfa14..c0238922a91 100644
--- a/storage/src/vespa/storage/storageserver/CMakeLists.txt
+++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt
@@ -11,7 +11,6 @@ vespa_add_library(storage_storageserver
documentapiconverter.cpp
fnetlistener.cpp
mergethrottler.cpp
- messageallocationtypes.cpp
messagesink.cpp
opslogger.cpp
priorityconverter.cpp
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
index eae51b90165..b214ef4539a 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
@@ -96,13 +96,6 @@ StorageTransportContext::StorageTransportContext(std::unique_ptr<RPCRequestWrapp
StorageTransportContext::~StorageTransportContext() { }
-const framework::MemoryAllocationType&
-CommunicationManager::getAllocationType(api::StorageMessage& msg) const
-{
- return _messageAllocTypes.getType(msg.getType().getId());
-}
-
-
void
CommunicationManager::receiveStorageReply(const std::shared_ptr<api::StorageReply>& reply)
{
@@ -296,8 +289,7 @@ CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, co
_configUri(configUri),
_closed(false),
_bucketResolver(std::make_unique<PlaceHolderBucketResolver>()),
- _docApiConverter(configUri, *_bucketResolver),
- _messageAllocTypes(_component.getMemoryManager())
+ _docApiConverter(configUri, *_bucketResolver)
{
_component.registerMetricUpdateHook(*this, framework::SecondTime(5));
_component.registerMetric(_metrics);
@@ -483,33 +475,9 @@ CommunicationManager::process(const std::shared_ptr<api::StorageMessage>& msg)
void
CommunicationManager::enqueue(const std::shared_ptr<api::StorageMessage> & msg)
{
- using MemoryToken = framework::MemoryToken;
assert(msg.get());
-
- const uint32_t memoryFootprint = msg->getMemoryFootprint();
- MemoryToken::UP token = _component.getMemoryManager().allocate(getAllocationType(*msg), memoryFootprint * 2,
- memoryFootprint * 2, msg->getPriority());
-
- if (token) {
- msg->setMemoryToken(std::move(token));
-
- LOG(spam, "Enq storage message %s, priority %d", msg->toString().c_str(), msg->getPriority());
- _eventQueue.enqueue(msg);
- } else {
- _metrics.failedDueToTooLittleMemory.inc();
- std::ostringstream ost;
- ost << "Failed to aquire " << (memoryFootprint * 2)
- << " bytes of memory to handle command of type "
- << msg->getType() << "\n";
- LOG(spam, "%s", ost.str().c_str());
- api::StorageCommand* cmd(dynamic_cast<api::StorageCommand*>(msg.get()));
-
- if (cmd) {
- std::shared_ptr<api::StorageReply> reply(cmd->makeReply());
- reply->setResult(api::ReturnCode(api::ReturnCode::BUSY, ost.str()));
- sendReply(reply);
- }
- }
+ LOG(spam, "Enq storage message %s, priority %d", msg->toString().c_str(), msg->getPriority());
+ _eventQueue.enqueue(msg);
}
bool
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h
index 4cf3f33e6ea..89b76583cb5 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.h
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.h
@@ -11,7 +11,6 @@
#pragma once
#include "communicationmanagermetrics.h"
-#include "messageallocationtypes.h"
#include "documentapiconverter.h"
#include <vespa/storage/common/storagelink.h>
#include <vespa/storage/common/storagecomponent.h>
@@ -171,10 +170,7 @@ private:
std::unique_ptr<BucketResolver> _bucketResolver;
DocumentApiConverter _docApiConverter;
framework::Thread::UP _thread;
- MessageAllocationTypes _messageAllocTypes;
- const framework::MemoryAllocationType&
- getAllocationType(api::StorageMessage& msg) const;
void updateMetrics(const MetricLockGuard &) override;
// Test needs access to configure() for live reconfig testing.
diff --git a/storage/src/vespa/storage/storageserver/framework.h b/storage/src/vespa/storage/storageserver/framework.h
index 724b4fe0944..f0ea1d71aa6 100644
--- a/storage/src/vespa/storage/storageserver/framework.h
+++ b/storage/src/vespa/storage/storageserver/framework.h
@@ -18,7 +18,6 @@
#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h>
#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h>
namespace storage {
@@ -27,7 +26,6 @@ struct Framework {
// Typedefs to simplify the remainder of the interface
typedef StorageComponentRegisterImpl CompReg;
typedef framework::defaultimplementation::RealClock RealClock;
- typedef framework::defaultimplementation::MemoryManager MemoryManager;
/**
* You can provide your own clock implementation. Useful in testing where
@@ -48,20 +46,10 @@ struct Framework {
*/
FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); }
- /**
- * Get the memory manager. Components that wants to print status of memory
- * manager need access to the actual implementation.
- */
- MemoryManager& getMemoryManager() { return _memoryManager; }
-
- void setMaximumMemoryUsage(uint64_t max);
-
private:
CompReg _componentRegister;
framework::Clock::UP _clock;
framework::defaultimplementation::ThreadPoolImpl _threadPool;
- framework::defaultimplementation::AllocationLogic* _memoryLogic;
- MemoryManager _memoryManager;
};
diff --git a/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp b/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp
deleted file mode 100644
index ac2d99a439c..00000000000
--- a/storage/src/vespa/storage/storageserver/messageallocationtypes.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "messageallocationtypes.h"
-#include <vespa/storageapi/messageapi/storagemessage.h>
-#include <vespa/vespalib/util/exceptions.h>
-#include <vespa/vespalib/stllike/asciistream.h>
-
-namespace storage {
-
-MessageAllocationTypes::MessageAllocationTypes(framework::MemoryManagerInterface& manager)
-{
- using api::MessageType;
- using framework::MemoryAllocationType;
-
- _types.resize(MessageType::MESSAGETYPE_MAX_ID);
- _types[MessageType::DOCBLOCK_ID] = &manager.registerAllocationType(MemoryAllocationType("MESSAGE_DOCBLOCK"));
- _types[MessageType::DOCBLOCK_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MESSAGE_DOCBLOCK_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GET", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::GET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::INTERNAL_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::INTERNAL"));
- _types[MessageType::INTERNAL_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::INTERNAL_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::PUT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::PUT", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::PUT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::PUT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::REMOVE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVE", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::REMOVE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::REVERT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REVERT"));
- _types[MessageType::REVERT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REVERT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::VISITOR_CREATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_CREATE", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::VISITOR_CREATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_CREATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::VISITOR_DESTROY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_DESTROY"));
- _types[MessageType::VISITOR_DESTROY_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_DESTROY_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::REQUESTBUCKETINFO_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REQUESTBUCKETINFO"));
- _types[MessageType::REQUESTBUCKETINFO_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REQUESTBUCKETINFO_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::NOTIFYBUCKETCHANGE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::NOTIFYBUCKETCHANGE"));
- _types[MessageType::NOTIFYBUCKETCHANGE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::NOTIFYBUCKETCHANGE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::CREATEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::CREATEBUCKET"));
- _types[MessageType::CREATEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::CREATEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::MERGEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MERGEBUCKET"));
- _types[MessageType::MERGEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MERGEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::DELETEBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DELETEBUCKET"));
- _types[MessageType::DELETEBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DELETEBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SETNODESTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETNODESTATE", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SETNODESTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETNODESTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETNODESTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETNODESTATE", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETNODESTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETNODESTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SETSYSTEMSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETSYSTEMSTATE", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SETSYSTEMSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETSYSTEMSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETSYSTEMSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETSYSTEMSTATE", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETSYSTEMSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETSYSTEMSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETBUCKETDIFF_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETDIFF", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETBUCKETDIFF_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETDIFF_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::APPLYBUCKETDIFF_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::APPLYBUCKETDIFF", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::APPLYBUCKETDIFF_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::APPLYBUCKETDIFF_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::VISITOR_INFO_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_INFO"));
- _types[MessageType::VISITOR_INFO_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::VISITOR_INFO_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SEARCHRESULT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SEARCHRESULT"));
- _types[MessageType::SEARCHRESULT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SEARCHRESULT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SPLITBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SPLITBUCKET"));
- _types[MessageType::SPLITBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SPLITBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::JOINBUCKETS_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::JOINBUCKETS"));
- _types[MessageType::JOINBUCKETS_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::JOINBUCKETS_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::SETBUCKETSTATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETBUCKETSTATE"));
- _types[MessageType::SETBUCKETSTATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::SETBUCKETSTATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::MULTIOPERATION_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MULTIOPERATION", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::MULTIOPERATION_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MULTIOPERATION_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::DOCUMENTSUMMARY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTSUMMARY"));
- _types[MessageType::DOCUMENTSUMMARY_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTSUMMARY_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::MAPVISITOR_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MAPVISITOR"));
- _types[MessageType::MAPVISITOR_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::MAPVISITOR_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::STATBUCKET_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::STATBUCKET", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::STATBUCKET_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::STATBUCKET_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::GETBUCKETLIST_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETLIST", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::GETBUCKETLIST_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::GETBUCKETLIST_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::DOCUMENTLIST_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTLIST"));
- _types[MessageType::DOCUMENTLIST_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::DOCUMENTLIST_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::UPDATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::UPDATE", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::UPDATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::UPDATE_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::EMPTYBUCKETS_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::EMPTYBUCKETS"));
- _types[MessageType::EMPTYBUCKETS_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::EMPTYBUCKETS_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::REMOVELOCATION_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVELOCATION", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::REMOVELOCATION_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::REMOVELOCATION_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::QUERYRESULT_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::QUERYRESULT"));
- _types[MessageType::QUERYRESULT_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::QUERYRESULT_REPLY", framework::MemoryAllocationType::FORCE_ALLOCATE));
- _types[MessageType::BATCHPUTREMOVE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHPUTREMOVE", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::BATCHPUTREMOVE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHPUTREMOVE_REPLY", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::BATCHDOCUMENTUPDATE_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHDOCUMENTUPDATE", framework::MemoryAllocationType::EXTERNAL_LOAD));
- _types[MessageType::BATCHDOCUMENTUPDATE_REPLY_ID] = &manager.registerAllocationType(MemoryAllocationType("MessageType::BATCHDOCUMENTUPDATE_REPLY", framework::MemoryAllocationType::EXTERNAL_LOAD));
-}
-
-const framework::MemoryAllocationType&
-MessageAllocationTypes::getType(uint32_t type) const {
- if (_types.size() > size_t(type) && _types[type] != 0) {
- return *_types[type];
- }
- vespalib::asciistream ost;
- ost << "No type registered with value " << type << ".";
- throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
-}
-
-} // storage
diff --git a/storage/src/vespa/storage/storageserver/messageallocationtypes.h b/storage/src/vespa/storage/storageserver/messageallocationtypes.h
deleted file mode 100644
index 8f7e8e4b3c3..00000000000
--- a/storage/src/vespa/storage/storageserver/messageallocationtypes.h
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::MessageAllocationTypes
- *
- * \brief Memory allocation types for messages in storage.
- */
-#pragma once
-
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
-#include <vector>
-
-namespace storage {
-
-class MessageAllocationTypes {
- std::vector<const framework::MemoryAllocationType*> _types;
-
-public:
- MessageAllocationTypes(framework::MemoryManagerInterface& manager);
-
- const framework::MemoryAllocationType& getType(uint32_t type) const;
-};
-
-}
-
diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp
index 855efaed6aa..ba1556bd3b9 100644
--- a/storage/src/vespa/storage/storageserver/storagenode.cpp
+++ b/storage/src/vespa/storage/storageserver/storagenode.cpp
@@ -7,7 +7,6 @@
#include "storagemetricsset.h"
#include "storagenodecontext.h"
-#include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h>
#include <vespa/storage/frameworkimpl/status/statuswebserver.h>
#include <vespa/storage/frameworkimpl/thread/deadlockdetector.h>
#include <vespa/storage/common/statusmetricconsumer.h>
@@ -117,7 +116,6 @@ StorageNode::initialize()
// and store them away, while having the config lock.
subscribeToConfigs();
- _context.getMemoryManager().setMaximumMemoryUsage(_serverConfig->memorytouse);
updateUpgradeFlag(*_clusterConfig);
// First update some basics that doesn't depend on anything else to be
@@ -157,11 +155,6 @@ StorageNode::initialize()
initializeNodeSpecific();
- _memoryStatusViewer.reset(new MemoryStatusViewer(
- _context.getMemoryManager(),
- _context.getComponentRegister().getMetricManager(),
- _context.getComponentRegister()));
-
_statusMetrics.reset(new StatusMetricConsumer(
_context.getComponentRegister(), _context.getComponentRegister().getMetricManager()));
_stateReporter.reset(new StateReporter(
@@ -259,14 +252,6 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard)
DIFFERWARN(clusterName, "Cannot alter cluster name of node live");
DIFFERWARN(nodeIndex, "Cannot alter node index of node live");
DIFFERWARN(isDistributor, "Cannot alter role of node live");
- {
- if (DIFFER(memorytouse)) {
- LOG(info, "Live config update: Memory to use changed from %" PRId64 " to %" PRId64 ".",
- oldC.memorytouse, newC.memorytouse);
- ASSIGN(memorytouse);
- _context.getMemoryManager().setMaximumMemoryUsage(newC.memorytouse);
- }
- }
_serverConfig.reset(new StorServerConfig(oldC));
_newServerConfig.reset();
(void)updated;
@@ -413,10 +398,6 @@ StorageNode::shutdown()
LOG(debug, "Deleting state reporter");
_stateReporter.reset();
}
- if (_memoryStatusViewer) {
- LOG(debug, "Deleting memory status viewer");
- _memoryStatusViewer.reset();
- }
if (_stateManager) {
LOG(debug, "Deleting state manager");
_stateManager.reset();
diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h
index 9b727ef3e0c..e9d3004be68 100644
--- a/storage/src/vespa/storage/storageserver/storagenode.h
+++ b/storage/src/vespa/storage/storageserver/storagenode.h
@@ -121,8 +121,6 @@ private:
// Depends on bucket databases and stop() functionality
std::unique_ptr<DeadLockDetector> _deadLockDetector;
- // Depends on dead lock detector and threadpool
- std::unique_ptr<MemoryStatusViewer> _memoryStatusViewer;
// Depends on metric manager
std::unique_ptr<StatusMetricConsumer> _statusMetrics;
// Depends on metric manager
diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp
index 2e1aa52e68d..75e1f12773f 100644
--- a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp
+++ b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp
@@ -2,31 +2,15 @@
#include "storagenodecontext.h"
-#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h>
-
namespace storage {
-using framework::defaultimplementation::AllocationLogic;
-using framework::defaultimplementation::PriorityMemoryLogic;
-
StorageNodeContext::StorageNodeContext(ComponentRegister::UP compReg, framework::Clock::UP clock)
: _componentRegister(std::move(compReg)),
_clock(std::move(clock)),
- _threadPool(*_clock),
- _memoryLogic(new PriorityMemoryLogic(*_clock, 1024 * 1024 * 1024)),
- _memoryManager(AllocationLogic::UP(_memoryLogic))
+ _threadPool(*_clock)
{
_componentRegister->setClock(*_clock);
_componentRegister->setThreadPool(_threadPool);
- _componentRegister->setMemoryManager(_memoryManager);
-}
-
-void
-StorageNodeContext::setMaximumMemoryUsage(uint64_t max)
-{
- using storage::framework::defaultimplementation::PriorityMemoryLogic;
- dynamic_cast<PriorityMemoryLogic*>(_memoryLogic)
- ->setMaximumMemoryUsage(max);
}
} // storage
diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.h b/storage/src/vespa/storage/storageserver/storagenodecontext.h
index 0149f975f63..eabca618bfb 100644
--- a/storage/src/vespa/storage/storageserver/storagenodecontext.h
+++ b/storage/src/vespa/storage/storageserver/storagenodecontext.h
@@ -18,7 +18,6 @@
#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h>
#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h>
namespace storage {
@@ -27,7 +26,6 @@ struct StorageNodeContext {
// Typedefs to simplify the remainder of the interface
typedef StorageComponentRegisterImpl ComponentRegister;
typedef framework::defaultimplementation::RealClock RealClock;
- typedef framework::defaultimplementation::MemoryManager MemoryManager;
/**
* Get the actual component register. Available as the actual type as the
@@ -42,14 +40,6 @@ struct StorageNodeContext {
*/
FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); }
- /**
- * Get the memory manager. Components that wants to print status of memory
- * manager need access to the actual implementation.
- */
- MemoryManager& getMemoryManager() { return _memoryManager; }
-
- void setMaximumMemoryUsage(uint64_t max);
-
protected:
// Initialization has been split in two as subclass needs to initialize
// component register before sending it on.
@@ -59,8 +49,6 @@ private:
ComponentRegister::UP _componentRegister;
framework::Clock::UP _clock;
framework::defaultimplementation::ThreadPoolImpl _threadPool;
- framework::defaultimplementation::AllocationLogic* _memoryLogic;
- MemoryManager _memoryManager;
};
diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp
index b91e38e035e..0d230745875 100644
--- a/storage/src/vespa/storage/visiting/visitor.cpp
+++ b/storage/src/vespa/storage/visiting/visitor.cpp
@@ -2,7 +2,6 @@
#include "visitor.h"
#include "visitormetrics.h"
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/generic/clock/timer.h>
#include <vespa/storageapi/message/datagram.h>
#include <vespa/storage/persistence/messages.h>
@@ -267,8 +266,7 @@ Visitor::Visitor(StorageComponent& component)
_id(),
_controlDestination(),
_dataDestination(),
- _documentSelection(),
- _memoryManager(0)
+ _documentSelection()
{
}
@@ -603,10 +601,6 @@ Visitor::start(api::VisitorId id, api::StorageMessage::Id cmdId,
_documentPriority = documentPriority;
_state = STATE_RUNNING;
- if (_memoryAllocType == 0) {
- _memoryAllocType = &_component.getMemoryManager()
- .getAllocationType("VISITOR_BUFFER");
- }
LOG(debug, "Starting visitor '%s' for %" PRIu64 " buckets from %" PRIu64 " to "
"%" PRIu64 ". First is %s. Max pending replies: %u, include "
@@ -803,17 +797,7 @@ Visitor::onCreateIteratorReply(
LOG(debug, "Visitor '%s' starting to visit bucket %s.",
_id.c_str(), bucketId.toString().c_str());
- framework::MemoryToken::UP token(
- _memoryManager->allocate(
- *_memoryAllocType, _docBlockSize, _docBlockSize, _priority));
- if (token.get() == 0) {
- // Not enough memory
- return;
- }
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token), bucket,
- bucketState.getIteratorId(),
- _docBlockSize));
+ auto cmd = std::make_shared<GetIterCommand>(bucket, bucketState.getIteratorId(), _docBlockSize);
cmd->setLoadType(_initiatingCmd->getLoadType());
cmd->getTrace().setLevel(_traceLevel);
cmd->setPriority(_priority);
@@ -1223,19 +1207,8 @@ Visitor::getIterators()
it = _bucketStates.erase(it);
continue;
}
- framework::MemoryToken::UP token(
- _memoryManager->allocate(
- *_memoryAllocType, _docBlockSize, _docBlockSize,
- _priority));
- if (token.get() == 0) {
- // Not enough memory
- return true;
- }
- std::shared_ptr<GetIterCommand> cmd(
- new GetIterCommand(std::move(token),
- bucketState.getBucket(),
- bucketState.getIteratorId(),
- _docBlockSize));
+ auto cmd = std::make_shared<GetIterCommand>(
+ bucketState.getBucket(), bucketState.getIteratorId(), _docBlockSize);
cmd->setLoadType(_initiatingCmd->getLoadType());
cmd->getTrace().setLevel(_traceLevel);
cmd->setPriority(_priority);
diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h
index 4436312032f..f84b105f1ed 100644
--- a/storage/src/vespa/storage/visiting/visitor.h
+++ b/storage/src/vespa/storage/visiting/visitor.h
@@ -327,7 +327,6 @@ private:
protected:
// These variables should not be altered after visitor starts. This not
// controlled by locks.
- const framework::MemoryAllocationType* _memoryAllocType;
VisitorMessageHandler* _messageHandler;
VisitorMessageSession::UP _messageSession;
documentapi::Priority::Value _documentPriority;
@@ -339,7 +338,6 @@ protected:
std::string _documentSelectionString;
std::unique_ptr<document::OrderingSpecification> _ordering;
vdslib::VisitorStatistics _visitorStatistics;
- framework::MemoryManagerInterface* _memoryManager;
bool isCompletedCalled() const { return _calledCompletedVisitor; }
@@ -365,8 +363,6 @@ public:
const api::StorageMessageAddress* getDataDestination() const
{ return _dataDestination.get(); } // Can't be null if attached
- void setAllocationType(const framework::MemoryAllocationType& mat)
- { _memoryAllocType = &mat; }
void setMaxPending(unsigned int maxPending)
{ _visitorOptions._maxPending = maxPending; }
@@ -384,8 +380,6 @@ public:
{ _docBlockTimeout = timeout; }
void setVisitorInfoTimeout(framework::MilliSecTime timeout)
{ _visitorInfoTimeout = timeout; }
- void setMemoryManager(framework::MemoryManagerInterface& mm)
- { _memoryManager = &mm; }
void setOwnNodeIndex(uint16_t nodeIndex) { _ownNodeIndex = nodeIndex; }
void setBucketSpace(document::BucketSpace bucketSpace) { _bucketSpace = bucketSpace; }
diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp
index deb38bb927d..f207c4436b1 100644
--- a/storage/src/vespa/storage/visiting/visitormanager.cpp
+++ b/storage/src/vespa/storage/visiting/visitormanager.cpp
@@ -7,7 +7,6 @@
#include "countvisitor.h"
#include "testvisitor.h"
#include "recoveryvisitor.h"
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storage/common/statusmessages.h>
#include <vespa/config/common/exceptions.h>
#include <vespa/documentapi/loadtypes/loadtypeset.h>
@@ -46,10 +45,6 @@ VisitorManager::VisitorManager(const config::ConfigUri & configUri,
_enforceQueueUse(false),
_visitorFactories(externalFactories)
{
- _component.getMemoryManager().registerAllocationType(
- framework::MemoryAllocationType(
- "VISITOR_BUFFER",
- framework::MemoryAllocationType::EXTERNAL_LOAD));
_configFetcher.subscribe<vespa::config::content::core::StorVisitorConfig>(configUri.getConfigId(), this);
_configFetcher.start();
_component.registerMetric(*_metrics);
diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp
index d3fed86b741..4743ac66054 100644
--- a/storage/src/vespa/storage/visiting/visitorthread.cpp
+++ b/storage/src/vespa/storage/visiting/visitorthread.cpp
@@ -2,7 +2,6 @@
#include "visitorthread.h"
#include "messages.h"
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/document/repo/documenttyperepo.h>
#include <vespa/document/select/bodyfielddetector.h>
#include <vespa/document/select/orderingselector.h>
@@ -100,9 +99,7 @@ VisitorThread::VisitorThread(uint32_t threadIndex,
_timeBetweenTicks(1000),
_component(componentRegister, getThreadName(threadIndex)),
_messageSessionFactory(messageSessionFac),
- _visitorFactories(visitorFactories),
- _memoryBufferAlloc(
- _component.getMemoryManager().getAllocationType("VISITOR_BUFFER"))
+ _visitorFactories(visitorFactories)
{
framework::MilliSecTime maxProcessingTime(30 * 1000);
framework::MilliSecTime waitTime(1000);
@@ -473,8 +470,6 @@ VisitorThread::onCreateVisitor(
cmd->getInstanceId().c_str(), errors.str().c_str());
break;
}
- visitor->setAllocationType(_memoryBufferAlloc);
- visitor->setMemoryManager(_component.getMemoryManager());
// Set visitor parameters
if (cmd->getMaximumPendingReplyCount() != 0) {
visitor->setMaxPending(cmd->getMaximumPendingReplyCount());
diff --git a/storage/src/vespa/storage/visiting/visitorthread.h b/storage/src/vespa/storage/visiting/visitorthread.h
index 1f1459ccb2b..97ca70f3761 100644
--- a/storage/src/vespa/storage/visiting/visitorthread.h
+++ b/storage/src/vespa/storage/visiting/visitorthread.h
@@ -90,7 +90,6 @@ class VisitorThread : public framework::Runnable,
framework::Thread::UP _thread;
VisitorMessageSessionFactory& _messageSessionFactory;
VisitorFactory::Map& _visitorFactories;
- const framework::MemoryAllocationType& _memoryBufferAlloc;
public:
VisitorThread(uint32_t threadIndex,
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
index 663a0ccdf4e..32f6c769e87 100644
--- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
+++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
@@ -12,7 +12,6 @@
#pragma once
#include "messagehandler.h"
-#include <vespa/storageframework/generic/memory/memorytoken.h>
#include <vespa/documentapi/loadtypes/loadtype.h>
#include <vespa/messagebus/routing/route.h>
#include <vespa/messagebus/trace.h>
@@ -335,7 +334,6 @@ private:
StorageMessage(const StorageMessage&);
mutable std::unique_ptr<TransportContext> _transportContext;
- std::unique_ptr<framework::MemoryToken> _memoryToken;
protected:
static Id generateMsgId();
@@ -375,10 +373,6 @@ public:
void setAddress(const StorageMessageAddress& address)
{ _address.reset(new StorageMessageAddress(address)); }
- void setMemoryToken(std::unique_ptr<framework::MemoryToken> token) {
- _memoryToken = std::move(token);
- }
-
/**
Returns the approximate memory footprint of a storage message.
By default, returns 50 bytes. This only needs to be overriden if the
diff --git a/storageframework/CMakeLists.txt b/storageframework/CMakeLists.txt
index 5a4b2e02372..a3414a8b05c 100644
--- a/storageframework/CMakeLists.txt
+++ b/storageframework/CMakeLists.txt
@@ -11,12 +11,10 @@ vespa_define_module(
src/vespa/storageframework/defaultimplementation
src/vespa/storageframework/defaultimplementation/clock
src/vespa/storageframework/defaultimplementation/component
- src/vespa/storageframework/defaultimplementation/memory
src/vespa/storageframework/defaultimplementation/thread
src/vespa/storageframework/generic
src/vespa/storageframework/generic/clock
src/vespa/storageframework/generic/component
- src/vespa/storageframework/generic/memory
src/vespa/storageframework/generic/metric
src/vespa/storageframework/generic/status
src/vespa/storageframework/generic/thread
@@ -28,7 +26,6 @@ vespa_define_module(
TESTS
src/tests
src/tests/clock
- src/tests/memory
src/tests/status
src/tests/thread
)
diff --git a/storageframework/src/tests/CMakeLists.txt b/storageframework/src/tests/CMakeLists.txt
index eb6d45330e8..0d604d64f74 100644
--- a/storageframework/src/tests/CMakeLists.txt
+++ b/storageframework/src/tests/CMakeLists.txt
@@ -5,7 +5,6 @@ vespa_add_executable(storageframework_testrunner_app TEST
DEPENDS
storageframework_testclock
storageframework_teststatus
- storageframework_testmemory
storageframework_testthread
)
diff --git a/storageframework/src/tests/memory/.gitignore b/storageframework/src/tests/memory/.gitignore
deleted file mode 100644
index 7e7c0fe7fae..00000000000
--- a/storageframework/src/tests/memory/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/.depend
-/Makefile
diff --git a/storageframework/src/tests/memory/CMakeLists.txt b/storageframework/src/tests/memory/CMakeLists.txt
deleted file mode 100644
index 591a78f1046..00000000000
--- a/storageframework/src/tests/memory/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(storageframework_testmemory
- SOURCES
- memorymanagertest.cpp
- memorystatetest.cpp
- DEPENDS
- storageframework
-)
diff --git a/storageframework/src/tests/memory/memorymanagertest.cpp b/storageframework/src/tests/memory/memorymanagertest.cpp
deleted file mode 100644
index 5424b8dea82..00000000000
--- a/storageframework/src/tests/memory/memorymanagertest.cpp
+++ /dev/null
@@ -1,397 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
-#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h>
-#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h>
-#include <vespa/vdstestlib/cppunit/macros.h>
-#include <vespa/vespalib/util/document_runnable.h>
-#include <vespa/vespalib/util/random.h>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-struct MemoryManagerTest : public CppUnit::TestFixture
-{
- void testBasics();
- void testCacheAllocation();
- void testStress();
-
- CPPUNIT_TEST_SUITE(MemoryManagerTest);
- CPPUNIT_TEST(testBasics);
- CPPUNIT_TEST(testCacheAllocation);
- CPPUNIT_TEST(testStress);
- CPPUNIT_TEST_SUITE_END();
-};
-
-CPPUNIT_TEST_SUITE_REGISTRATION(MemoryManagerTest);
-
-void
-MemoryManagerTest::testBasics()
-{
- uint64_t maxMemory = 1000;
- RealClock clock;
- SimpleMemoryLogic* logic = new SimpleMemoryLogic(clock, maxMemory);
- AllocationLogic::UP allLogic(std::move(logic));
- MemoryManager manager(std::move(allLogic));
-
- const MemoryAllocationType& putAlloc(manager.registerAllocationType(
- MemoryAllocationType("put", MemoryAllocationType::EXTERNAL_LOAD)));
- const MemoryAllocationType& getAlloc(manager.registerAllocationType(
- MemoryAllocationType("get", MemoryAllocationType::EXTERNAL_LOAD)));
- const MemoryAllocationType& bufAlloc(manager.registerAllocationType(
- MemoryAllocationType("buffer")));
- const MemoryAllocationType& cacheAlloc(manager.registerAllocationType(
- MemoryAllocationType("cache", MemoryAllocationType::CACHE)));
- const MemoryState& state(logic->getState());
- const MemoryState::SnapShot& current(state.getCurrentSnapshot());
- // Basics
- {
- // * Getting a token, and release it back with correct behavior
- framework::MemoryToken::UP put = manager.allocate(putAlloc,
- 0, 100, 80);
- CPPUNIT_ASSERT(put.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(100), put->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(100), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(900), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
-
- // * Do the same while not being empty. Different type.
- framework::MemoryToken::UP get = manager.allocate(getAlloc,
- 30, 200, 50);
- CPPUNIT_ASSERT(get.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(200), get->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(300), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(700), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
-
- // * Do the same while not being empty. Same type.
- framework::MemoryToken::UP get2 = manager.allocate(
- getAlloc,
- 70,
- 150,
- 60);
-
- CPPUNIT_ASSERT(get2.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(150), get2->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(450), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(550), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
-
- // Non-external load
- // * Getting minimum when going beyond 80% full
- {
- framework::MemoryToken::UP filler = manager.allocate(putAlloc,
- 795, 795, 90);
- framework::MemoryToken::UP resize = manager.allocate(
- bufAlloc, 10, 90, 80);
- CPPUNIT_ASSERT(resize.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(10), resize->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(805), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(195), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
-
- // Non-external load
- // * Getting up to threshold if hitting it
- {
- framework::MemoryToken::UP filler = manager.allocate(putAlloc,
- 750, 750, 90);
- framework::MemoryToken::UP resize = manager.allocate(
- bufAlloc, 10, 90, 80);
- CPPUNIT_ASSERT(resize.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(50), resize->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(800), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(200), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
-
- // External load
- {
- // * Stopped when going beyond 80% full
- framework::MemoryToken::UP filler = manager.allocate(putAlloc,
- 795, 795, 90);
- framework::MemoryToken::UP put = manager.allocate(putAlloc,
- 10, 100, 80);
- CPPUNIT_ASSERT(put.get() == 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(795), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(205), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
-
- // External load
- {
- // * Getting up to threshold if hitting it
- framework::MemoryToken::UP filler = manager.allocate(putAlloc,
- 750, 750, 90);
- framework::MemoryToken::UP put = manager.allocate(putAlloc,
- 10, 100, 80);
- CPPUNIT_ASSERT(put.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(50), put->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(800), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(200), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
-
- // Test caching..
- {
- // Cache paradigm:
- // Allocate a token taking up no space at all.
- // Give it to your ReduceMemoryUsageInterface implementation.
- // Run resize on your token in that implementation to get memory and
- // return memory. That way locking should be easy when needed.
- struct ReduceI : public framework::ReduceMemoryUsageInterface {
- framework::MemoryToken::UP _token;
-
- uint64_t reduceMemoryConsumption(const MemoryToken& token, uint64_t reduceBy) override {
- assert(&token == _token.get());
- (void) &token;
- assert(_token->getSize() >= reduceBy);
- return reduceBy;
- }
- };
- ReduceI reducer;
- framework::MemoryToken::UP cache = manager.allocate(cacheAlloc,
- 0, 0, 0, &reducer);
- CPPUNIT_ASSERT(cache.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), cache->getSize());
- reducer._token = std::move(cache);
- for (uint32_t i=1; i<=50; ++i) {
- bool success = reducer._token->resize(i * 10, i * 10);
- CPPUNIT_ASSERT_EQUAL(true, success);
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(500), reducer._token->getSize());
-
- // * Ordered to free space
- framework::MemoryToken::UP put = manager.allocate(putAlloc,
- 600, 600, 80);
- CPPUNIT_ASSERT_EQUAL_MSG(manager.toString(),
- uint64_t(400), reducer._token->getSize());
- CPPUNIT_ASSERT_EQUAL_MSG(manager.toString(),
- uint64_t(600), put->getSize());
- }
- CPPUNIT_ASSERT_EQUAL_MSG(state.toString(true),
- uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL_MSG(state.toString(true),
- uint64_t(0), current.getUserCount());
-
- // Test merge and tracking of allocation counts with merge, by doing
- // operations with tokens and see that user count and used size
- // correctly go back to zero.
- {
- framework::MemoryToken::UP tok1(
- manager.allocate(putAlloc, 5, 5, 40));
- framework::MemoryToken::UP tok2(
- manager.allocate(putAlloc, 10, 10, 40));
- framework::MemoryToken::UP tok3(
- manager.allocate(putAlloc, 20, 20, 40));
- framework::MemoryToken::UP tok4(
- manager.allocate(putAlloc, 40, 40, 40));
- framework::MemoryToken::UP tok5(
- manager.allocate(putAlloc, 80, 80, 40));
- framework::MemoryToken::UP tok6(
- manager.allocate(putAlloc, 1, 1, 40));
- framework::MemoryToken::UP tok7(
- manager.allocate(putAlloc, 3, 3, 40));
- }
-}
-
-void
-MemoryManagerTest::testCacheAllocation()
-{
- uint64_t maxMemory = 3000;
-
- RealClock clock;
- SimpleMemoryLogic::UP logic(new PriorityMemoryLogic(clock, maxMemory));
- logic->setCacheThreshold(1.0);
-
- AllocationLogic::UP allLogic(std::move(logic));
- MemoryManager manager(std::move(allLogic));
-
- const MemoryAllocationType& putAlloc(manager.registerAllocationType(
- MemoryAllocationType("put", MemoryAllocationType::EXTERNAL_LOAD)));
- const MemoryAllocationType& cacheAlloc(manager.registerAllocationType(
- MemoryAllocationType("cache", MemoryAllocationType::CACHE)));
-
- framework::MemoryToken::UP token =
- manager.allocate(putAlloc,
- 50,
- 50,
- 127);
-
- CPPUNIT_ASSERT_EQUAL(50, (int)token->getSize());
-
- framework::MemoryToken::UP token2 =
- manager.allocate(cacheAlloc,
- 1000,
- 2000,
- 127);
-
- CPPUNIT_ASSERT_EQUAL(2000, (int)token2->getSize());
-
- token2->resize(2000, 3000);
-
- CPPUNIT_ASSERT_EQUAL(2950, (int)token2->getSize());
-}
-
-namespace {
-struct MemoryManagerLoadGiver : public document::Runnable,
- public ReduceMemoryUsageInterface
-{
- MemoryManager& _manager;
- const framework::MemoryAllocationType& _type;
- uint8_t _priority;
- uint32_t _minMem;
- uint32_t _maxMem;
- uint32_t _failed;
- uint32_t _ok;
- uint32_t _reduced;
- using MemoryTokenUP = std::unique_ptr<MemoryToken>;
- std::vector<MemoryTokenUP> _tokens;
- vespalib::Lock _cacheLock;
-
- MemoryManagerLoadGiver(
- MemoryManager& manager,
- const framework::MemoryAllocationType& type,
- uint8_t priority,
- uint32_t minMem,
- uint32_t maxMem,
- uint32_t tokensToKeep)
- : _manager(manager),
- _type(type),
- _priority(priority),
- _minMem(minMem),
- _maxMem(maxMem),
- _failed(0),
- _ok(0),
- _reduced(0),
- _tokens(tokensToKeep)
- {
- }
-
- uint64_t reduceMemoryConsumption(const MemoryToken&, uint64_t reduceBy) override {
- ++_reduced;
- return reduceBy;
- }
-
- void run() override {
- ReduceMemoryUsageInterface* reducer = 0;
- if (_type.isCache()) reducer = this;
- vespalib::RandomGen randomizer;
- while (running()) {
- vespalib::Lock lock(_cacheLock);
- framework::MemoryToken::UP token = _manager.allocate(
- _type, _minMem, _maxMem, _priority, reducer);
- if (token.get() == 0) {
- ++_failed;
- } else {
- ++_ok;
- }
- uint32_t index = randomizer.nextUint32(0, _tokens.size() - 1);
- _tokens[index] = MemoryTokenUP(token.release());
- }
- }
-};
-}
-
-void
-MemoryManagerTest::testStress()
-{
- uint64_t stressTimeMS = 1 * 1000;
- uint64_t maxMemory = 1 * 1024 * 1024;
- RealClock clock;
- AllocationLogic::UP logic(new PriorityMemoryLogic(clock, maxMemory));
- MemoryManager manager(std::move(logic));
-
- FastOS_ThreadPool pool(128 * 1024);
- std::vector<MemoryManagerLoadGiver*> loadGivers;
- for (uint32_t type = 0; type < 5; ++type) {
- const MemoryAllocationType* allocType = 0;
- uint32_t min = 1000, max = 5000;
- if (type == 0) {
- allocType = &manager.registerAllocationType(MemoryAllocationType(
- "default"));
- } else if (type == 1) {
- allocType = &manager.registerAllocationType(MemoryAllocationType(
- "external", MemoryAllocationType::EXTERNAL_LOAD));
- } else if (type == 2) {
- allocType = &manager.registerAllocationType(MemoryAllocationType(
- "forced", MemoryAllocationType::FORCE_ALLOCATE));
- } else if (type == 3) {
- allocType = &manager.registerAllocationType(MemoryAllocationType(
- "forcedExternal", MemoryAllocationType::FORCE_ALLOCATE
- | MemoryAllocationType::EXTERNAL_LOAD));
- } else if (type == 4) {
- allocType = &manager.registerAllocationType(MemoryAllocationType(
- "cache", MemoryAllocationType::CACHE));
- max = 30000;
- }
- for (int priority = 0; priority < 256; priority += 8) {
- loadGivers.push_back(new MemoryManagerLoadGiver(
- manager, *allocType, priority, min, max, 10));
- loadGivers.back()->start(pool);
- }
- FastOS_Thread::Sleep(stressTimeMS);
- }
- FastOS_Thread::Sleep(5 * stressTimeMS);
- uint64_t okTotal = 0, failedTotal = 0, reducedTotal = 0;
- for (uint32_t i = 0; i < loadGivers.size(); i++) {
- /*
- fprintf(stderr, "%d %s-%u: Failed %d, ok %d, reduced %d\n",
- i, loadGivers[i]->_type.getName().c_str(),
- uint32_t(loadGivers[i]->_priority),
- loadGivers[i]->_failed, loadGivers[i]->_ok,
- loadGivers[i]->_reduced); // */
- okTotal += loadGivers[i]->_ok;
- failedTotal += loadGivers[i]->_failed;
- reducedTotal += loadGivers[i]->_reduced;
- }
- for (uint32_t i = 0; i < loadGivers.size(); i++) loadGivers[i]->stop();
- for (uint32_t i = 0; i < loadGivers.size(); i++) loadGivers[i]->join();
- pool.Close();
-
- /*
- bool verbose = false;
- std::cerr << "\n\nMemory allocations at end of load:\n";
- manager.print(std::cerr, verbose, ""); // */
-
- for (uint32_t i = 0; i < loadGivers.size(); i++) {
- loadGivers[i]->_tokens.clear();
- }
- for (uint32_t i = 0; i < loadGivers.size(); i++) {
- delete loadGivers[i];
- }
- loadGivers.clear();
-
- //std::cerr << "\n\nMemory allocations at end of testl:\n";
- //manager.print(std::cerr, verbose, "");
-
- std::cerr << "\n Managed " << std::fixed
- << (okTotal / (stressTimeMS / 1000))
- << " ok, " << (failedTotal / (stressTimeMS / 1000))
- << " failed and " << (reducedTotal / (stressTimeMS / 1000))
- << " reduced allocations/s.\n ";
-
- MemoryState state(clock, 1);
- manager.getState(state);
- const MemoryState::SnapShot& current(state.getCurrentSnapshot());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSizeIgnoringCache());
-}
-
-} // defaultimplementation
-} // framework
-} // storage
diff --git a/storageframework/src/tests/memory/memorystatetest.cpp b/storageframework/src/tests/memory/memorystatetest.cpp
deleted file mode 100644
index cd565718632..00000000000
--- a/storageframework/src/tests/memory/memorystatetest.cpp
+++ /dev/null
@@ -1,172 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorystate.h>
-#include <vespa/vdstestlib/cppunit/macros.h>
-#include <vespa/vespalib/util/exceptions.h>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-struct MemoryStateTest : public CppUnit::TestFixture
-{
-
- void testBasics();
-
- CPPUNIT_TEST_SUITE(MemoryStateTest);
- CPPUNIT_TEST(testBasics); // Fails sometimes, test needs rewrite.
- CPPUNIT_TEST_SUITE_END();
-};
-
-CPPUNIT_TEST_SUITE_REGISTRATION(MemoryStateTest);
-
-class SimpleMemoryManager : public framework::MemoryManagerInterface
-{
-private:
- std::map<std::string, framework::MemoryAllocationType> _types;
-
-public:
- void setMaximumMemoryUsage(uint64_t max) override { (void) max; }
-
- const framework::MemoryAllocationType&
- registerAllocationType(const framework::MemoryAllocationType& type) override {
- _types[type.getName()] = type;
- return _types[type.getName()];
- }
-
- const framework::MemoryAllocationType&
- getAllocationType(const std::string& name) const override {
- std::map<std::string, framework::MemoryAllocationType>::const_iterator iter =
- _types.find(name);
-
- if (iter == _types.end()) {
- throw vespalib::IllegalArgumentException("Allocation type not found: " + name);
- }
-
- return iter->second;
- }
-
- std::vector<const MemoryAllocationType*> getAllocationTypes() const override {
- std::vector<const MemoryAllocationType*> types;
- for(std::map<std::string, framework::MemoryAllocationType>
- ::const_iterator it = _types.begin(); it != _types.end(); ++it)
- {
- types.push_back(&it->second);
- }
- return types;
- }
-
- framework::MemoryToken::UP allocate(const framework::MemoryAllocationType&,
- uint64_t,
- uint64_t,
- uint8_t,
- framework::ReduceMemoryUsageInterface*) override
- {
- return framework::MemoryToken::UP();
- }
-
- uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override {
- (void) priority;
- return 0;
- }
-};
-
-void
-MemoryStateTest::testBasics()
-{
- SimpleMemoryManager manager;
-
- const MemoryAllocationType& putAlloc(manager.registerAllocationType(
- MemoryAllocationType("MESSAGE_PUT", MemoryAllocationType::EXTERNAL_LOAD)));
- const MemoryAllocationType& getAlloc(manager.registerAllocationType(
- MemoryAllocationType("MESSAGE_GET", MemoryAllocationType::EXTERNAL_LOAD)));
- const MemoryAllocationType& blockAlloc(manager.registerAllocationType(
- MemoryAllocationType("MESSAGE_DOCBLOCK")));
- const MemoryAllocationType& databaseAlloc(manager.registerAllocationType(
- MemoryAllocationType("DATABASE")));
- const MemoryAllocationType& cacheAlloc(manager.registerAllocationType(
- MemoryAllocationType("SLOTFILE_CACHE", MemoryAllocationType::CACHE)));
-
- uint32_t maxMemory = 1024;
-
- RealClock clock;
- MemoryState state1(clock, maxMemory);
- MemoryState state2(clock, maxMemory);
-
- state1.setMinJumpToUpdateMax(50);
-
- state1.addToEntry(putAlloc, 100, 10,
- MemoryState::GOT_MAX, false);
- state1.addToEntry(putAlloc, 100, 60,
- MemoryState::GOT_MAX, false);
- state1.addToEntry(blockAlloc,
- 200, 20,
- MemoryState::GOT_MIN, false);
- state1.addToEntry(getAlloc, 0, 15,
- MemoryState::DENIED, false, 0);
- state1.addToEntry(databaseAlloc, 150, 0,
- MemoryState::DENIED, true, 1);
- state1.addToEntry(cacheAlloc, 45, 0,
- MemoryState::GOT_MAX, true, 1);
-
- state2.addToEntry(putAlloc, 50, 10,
- MemoryState::GOT_MIN, false);
- state2.addToEntry(putAlloc, 20, 40,
- MemoryState::GOT_MIN, false);
-
- state1.removeFromEntry(databaseAlloc, 25, 0, 0);
- state1.removeFromEntry(putAlloc, 100, 60);
-
- MemoryState::SnapShot state3;
- state3 = state1.getMaxSnapshot();
- state3 += state2.getMaxSnapshot();
-
- std::string expected;
- expected =
- "\n"
- "MemoryState(Max memory: 1024) {\n"
- " Current: SnapShot(Used 470, w/o cache 425) {\n"
- " Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n"
- " DATABASE(0): Used(125 B / 1) Stats(1, 0, 0, 1, 1)\n"
- " MESSAGE_DOCBLOCK(20): Used(200 B / 1) Stats(1, 0, 1, 0, 0)\n"
- " MESSAGE_GET(15): Used(0 B / 0) Stats(1, 0, 0, 1, 0)\n"
- " MESSAGE_PUT(10): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n"
- " MESSAGE_PUT(60): Used(0 B / 0) Stats(1, 1, 0, 0, 0)\n"
- " SLOTFILE_CACHE(0): Used(45 B / 1) Stats(1, 1, 0, 0, 1)\n"
- " }\n"
- " Max: SnapShot(Used 550, w/o cache 550) {\n"
- " Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n"
- " DATABASE(0): Used(150 B / 1) Stats(1, 0, 0, 1, 1)\n"
- " MESSAGE_DOCBLOCK(20): Used(200 B / 1) Stats(1, 0, 1, 0, 0)\n"
- " MESSAGE_GET(15): Used(0 B / 0) Stats(1, 0, 0, 1, 0)\n"
- " MESSAGE_PUT(10): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n"
- " MESSAGE_PUT(60): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n"
- " }\n"
- "}";
-
- CPPUNIT_ASSERT_EQUAL(expected, "\n" + state1.toString(true));
- expected = "\n"
-"MemoryState(Max memory: 1024) {\n"
-" Current: SnapShot(Used 70, w/o cache 70) {\n"
-" Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n"
-" MESSAGE_PUT(10): Used(50 B / 1) Stats(1, 0, 1, 0, 0)\n"
-" MESSAGE_PUT(40): Used(20 B / 1) Stats(1, 0, 1, 0, 0)\n"
-" }\n"
-"}";
- CPPUNIT_ASSERT_EQUAL(expected, "\n" + state2.toString(true));
- expected = "\n"
-"SnapShot(Used 550, w/o cache 550) {\n"
-" Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n"
-" DATABASE(0): Used(150 B / 1) Stats(1, 0, 0, 1, 1)\n"
-" MESSAGE_DOCBLOCK(20): Used(200 B / 1) Stats(1, 0, 1, 0, 0)\n"
-" MESSAGE_GET(15): Used(0 B / 0) Stats(1, 0, 0, 1, 0)\n"
-" MESSAGE_PUT(10): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n"
-" MESSAGE_PUT(60): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n"
-"}";
- CPPUNIT_ASSERT_EQUAL(expected, "\n" + state3.toString(true));
-}
-
-} // defaultimplementation
-} // framework
-} // storage
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt b/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt
index 25e7a9812cf..ea6a7e15543 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt
@@ -3,7 +3,6 @@ vespa_add_library(storageframework_defaultimplementation
SOURCES
$<TARGET_OBJECTS:storageframework_clockimpl>
$<TARGET_OBJECTS:storageframework_componentimpl>
- $<TARGET_OBJECTS:storageframework_memoryimpl>
$<TARGET_OBJECTS:storageframework_threadimpl>
INSTALL lib64
DEPENDS
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp
index a03f79b008d..84b091d25e3 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp
@@ -63,16 +63,6 @@ ComponentRegisterImpl::setMetricManager(metrics::MetricManager& mm)
}
void
-ComponentRegisterImpl::setMemoryManager(MemoryManagerInterface& mm)
-{
- vespalib::LockGuard lock(_componentLock);
- _memoryManager = &mm;
- for (uint32_t i=0; i<_components.size(); ++i) {
- _components[i]->setMemoryManager(mm);
- }
-}
-
-void
ComponentRegisterImpl::setClock(Clock& c)
{
vespalib::LockGuard lock(_componentLock);
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h
index 7319a1e7d57..3e3dd08e1df 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h
@@ -71,7 +71,6 @@ public:
void requestShutdown(vespalib::stringref reason) override;
void setMetricManager(metrics::MetricManager&);
- void setMemoryManager(MemoryManagerInterface&);
void setClock(Clock&);
void setThreadPool(ThreadPool&);
void setUpgradeFlag(UpgradeFlags flag);
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp
index 0f4d0ebe070..5efd638ec26 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp
@@ -7,13 +7,9 @@ namespace storage::framework::defaultimplementation {
TestComponentRegister::TestComponentRegister(ComponentRegisterImpl::UP compReg)
: _compReg(std::move(compReg)),
_clock(),
- _threadPool(_clock),
- _memoryManager()
+ _threadPool(_clock)
{
assert(_compReg.get() != 0);
- // Set a memory manager, so users can register memory types and
- // ask for memory.
- _compReg->setMemoryManager(_memoryManager);
// Set a fake clock, giving test control of clock
_compReg->setClock(_clock);
// Set a thread pool so components can make threads in tests.
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h
index 5226765dc40..4983d5dcdfc 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h
@@ -15,7 +15,6 @@
#include "componentregisterimpl.h"
#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h>
-#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h>
#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h>
namespace storage::framework::defaultimplementation {
@@ -24,7 +23,6 @@ class TestComponentRegister {
ComponentRegisterImpl::UP _compReg;
FakeClock _clock;
ThreadPoolImpl _threadPool;
- NoMemoryManager _memoryManager;
public:
TestComponentRegister(ComponentRegisterImpl::UP compReg);
@@ -34,7 +32,6 @@ public:
FakeClock& getClock() { return _clock; }
ThreadPoolImpl& getThreadPoolImpl() { return _threadPool; }
FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); }
- NoMemoryManager& getMemoryManager() { return _memoryManager; }
};
}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore b/storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore
deleted file mode 100644
index 7e7c0fe7fae..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/.depend
-/Makefile
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt b/storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt
deleted file mode 100644
index e0aa10a6716..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(storageframework_memoryimpl OBJECT
- SOURCES
- memorymanager.cpp
- simplememorylogic.cpp
- memorystate.cpp
- prioritymemorylogic.cpp
- nomemorymanager.cpp
- DEPENDS
-)
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h
deleted file mode 100644
index 5af2c9ff929..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <storage/memorymanager/memorymanager.h>
-#include <storage/memorymanager/memorystate.h>
-
-namespace storage {
-
-class EmptyMemoryLogic : public AllocationLogic
-{
-private:
- MemoryState _state;
-
-public:
- EmptyMemoryLogic() : _state(100) {}
-
- virtual void getState(MemoryState& state, bool resetMax) {
- state = _state;
- if (resetMax) _state.resetMax();
- }
-
- virtual MemoryToken::UP allocate(
- const AllocationType& type,
- uint64_t, uint64_t max,
- storage::api::StorageMessage::Priority p,
- ReduceMemoryUsageInterface* = 0) {
- return MemoryToken::UP(
- new MemoryToken(*this, type, max, p));
- }
-
- virtual bool resize(MemoryToken& token, uint64_t min, uint64_t max) {
- setTokenSize(token, max);
- return true;
- }
-
- virtual void freeToken(MemoryToken&) {}
-
- virtual void print(std::ostream& out, bool verbose,
- const std::string& indent) const
- {
- (void) verbose; (void) indent;
- out << "EmptyMemoryLogic()";
- }
-};
-
-} // storage
-
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp
deleted file mode 100644
index 5af222d4b36..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp
+++ /dev/null
@@ -1,157 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memorymanager.h"
-#include "memorystate.h"
-#include <vespa/vespalib/util/exceptions.h>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-MemoryTokenImpl::MemoryTokenImpl(AllocationLogic& logic,
- const MemoryAllocationType& type,
- uint64_t allocated,
- uint8_t p,
- ReduceMemoryUsageInterface* reducer)
- : _logic(logic),
- _reducer(reducer),
- _currentlyAllocated(allocated),
- _allocCount(1),
- _type(type),
- _priority(p)
-{
-}
-
-MemoryTokenImpl::~MemoryTokenImpl()
-{
- _logic.freeToken(*this);
-}
-
-bool
-MemoryTokenImpl::resize(uint64_t min, uint64_t max)
-{
- return _logic.resize(*this, min, max);
-}
-
-void
-MemoryTokenImpl::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- (void) verbose; (void) indent;
- out << "MemoryToken(" << _type.getName() << ": Allocated(" << _allocCount << " - "
- << _currentlyAllocated << ")";
-}
-
-AllocationLogic::~AllocationLogic()
-{
-}
-
-MemoryToken::UP
-AllocationLogic::allocate(const MemoryAllocationType& type,
- uint64_t min,
- uint64_t max,
- uint8_t priority,
- ReduceMemoryUsageInterface* reducer)
-{
- MemoryToken::UP token(allocate(type, priority, reducer));
- assert(token.get());
- if (!resize(*token, min, max, 1)) token.reset();
- return token;
-}
-
-bool
-AllocationLogic::resize(MemoryTokenImpl& token, uint64_t min, uint64_t max)
-{
- return resize(token, min, max, 0);
-}
-
-MemoryManager::MemoryManager(AllocationLogic::UP logic)
- : _logic(std::move(logic))
-{
- if (_logic.get() == 0) {
- throw vespalib::IllegalArgumentException(
- "Needs a real logic class to run. (Got null pointer)",
- VESPA_STRLOC);
- }
-}
-
-MemoryManager::~MemoryManager()
-{
-}
-
-void
-MemoryManager::setMaximumMemoryUsage(uint64_t max)
-{
- _logic->setMaximumMemoryUsage(max);
-}
-
-void
-MemoryManager::getState(MemoryState& state, bool resetMax)
-{
- return _logic->getState(state, resetMax);
-}
-
-const MemoryAllocationType&
-MemoryManager::registerAllocationType(const MemoryAllocationType& type)
-{
- vespalib::LockGuard lock(_typeLock);
- _types[type.getName()] = MemoryAllocationType::UP(
- new MemoryAllocationType(type));
- return *_types[type.getName()];
-}
-
-const MemoryAllocationType&
-MemoryManager::getAllocationType(const std::string& name) const
-{
- vespalib::LockGuard lock(_typeLock);
- std::map<std::string, MemoryAllocationType::UP>::const_iterator it(
- _types.find(name));
- if (it == _types.end()) {
- throw vespalib::IllegalArgumentException(
- "Allocation type not found: " + name, VESPA_STRLOC);
- }
- return *it->second;
-}
-
-std::vector<const MemoryAllocationType*>
-MemoryManager::getAllocationTypes() const
-{
- vespalib::LockGuard lock(_typeLock);
- std::vector<const MemoryAllocationType*> types;
- for(std::map<std::string, MemoryAllocationType::UP>::const_iterator it
- = _types.begin(); it != _types.end(); ++it)
- {
- types.push_back(it->second.get());
- }
- return types;
-}
-
-MemoryToken::UP
-MemoryManager::allocate(const MemoryAllocationType& type,
- uint64_t min,
- uint64_t max,
- uint8_t p,
- ReduceMemoryUsageInterface* i)
-{
- return _logic->allocate(type, min, max, p, i);
-}
-
-uint64_t
-MemoryManager::getMemorySizeFreeForPriority(uint8_t priority) const
-{
- return _logic->getMemorySizeFreeForPriority(priority);
-}
-
-void
-MemoryManager::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- (void) verbose; (void) indent;
- out << "Memory Manager {" << "\n" << indent << " ";
- _logic->print(out, verbose, indent + " ");
- out << "\n" << indent << "}";
-}
-
-} // defaultimplementation
-} // framework
-} // storage
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h
deleted file mode 100644
index 86a2920095e..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h
+++ /dev/null
@@ -1,156 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::MemoryManager
- *
- * \brief Utility for tracking memory usage on distributor and storage nodes.
- *
- * The memory manager is responsible for limiting the memory allocated by
- * various users within VDS storage and distributor nodes, such that these
- * nodes don't use more memory than they can use, to avoid swapping.
- *
- * It will produce a status page to view big memory users, and give some
- * historic data. It will track memory users and give them less and less memory
- * as closer we are to utilizing all memory we are able to use. When getting
- * close to full it will deny memory allocations to incoming commands that wants
- * to use additional memory in able to complete the operation.
- *
- * The main class here defines the interface the client has to worry about. It
- * should thus not point to the implementation in any way.
- *
- */
-
-#pragma once
-
-#include <map>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
-#include <vespa/vespalib/util/printable.h>
-#include <vespa/vespalib/util/sync.h>
-
-namespace storage::framework::defaultimplementation {
-
-class MemoryManager;
-class AllocationLogic;
-class MemoryState;
-
-class MemoryTokenImpl : public vespalib::Printable,
- public MemoryToken
-{
- friend class AllocationLogic;
-
- AllocationLogic& _logic;
- ReduceMemoryUsageInterface* _reducer;
- uint64_t _currentlyAllocated;
- uint32_t _allocCount;
- const MemoryAllocationType& _type;
- uint8_t _priority;
-
-public:
- typedef std::unique_ptr<MemoryTokenImpl> UP;
-
- MemoryTokenImpl(const MemoryTokenImpl &) = delete;
- MemoryTokenImpl & operator = (const MemoryTokenImpl &) = delete;
- MemoryTokenImpl(AllocationLogic& logic,
- const MemoryAllocationType& type,
- uint64_t allocated,
- uint8_t priority,
- ReduceMemoryUsageInterface* = 0);
-
- ~MemoryTokenImpl();
-
- uint64_t getSize() const override { return _currentlyAllocated; }
- uint64_t getAllocationCount() const { return _allocCount; }
- const MemoryAllocationType& getType() const { return _type; }
- ReduceMemoryUsageInterface* getReducer() const { return _reducer; }
-
- uint8_t getPriority() const { return _priority; }
-
- bool resize(uint64_t min, uint64_t max) override;
-
- void print(std::ostream& out, bool verbose,
- const std::string& indent) const override;
-};
-
-class AllocationLogic : public vespalib::Printable
-{
-protected:
- /**
- * MemoryTokens are friends with this class, such that logic classes
- * can use this function to alter token size, without function for that
- * being public.
- */
- void setTokenSize(MemoryTokenImpl& token, uint64_t size)
- { token._currentlyAllocated = size; }
-
- virtual MemoryToken::UP allocate(const MemoryAllocationType&,
- uint8_t priority,
- ReduceMemoryUsageInterface*) = 0;
- virtual bool resize(MemoryToken& token, uint64_t min, uint64_t max,
- uint32_t allocationCounts) = 0;
-public:
- typedef std::unique_ptr<AllocationLogic> UP;
- virtual ~AllocationLogic() = 0;
-
- virtual void setMaximumMemoryUsage(uint64_t max) = 0;
-
- virtual void getState(MemoryState&, bool resetMax = false) = 0;
-
- /**
- * Decide how much to allocate for this request. Should be between min
- * and max, unless it's of a type that can be denied (such as external
- * requests), in which case we can also deny allocation by returning a null
- * token.
- */
- MemoryToken::UP allocate(const MemoryAllocationType&,
- uint64_t min,
- uint64_t max,
- uint8_t priority,
- ReduceMemoryUsageInterface* = 0);
- /**
- * Resize the size in a token. If more memory is requested, then it might
- * fail. The sizes given in min and max is given as total min and max,
- * including any memory you may already have. If successful, the logic will
- * have added this size to the token passed in.
- */
- bool resize(MemoryTokenImpl& token, uint64_t min, uint64_t max);
-
- // Called by token destructor to free up tracked resources
- virtual void freeToken(MemoryTokenImpl& token) = 0;
-
- virtual uint64_t getMemorySizeFreeForPriority(uint8_t priority) const = 0;
-
- // vespalib::Printable implementation
- virtual void print(std::ostream& out, bool verbose,
- const std::string& indent) const override = 0;
-};
-
-class MemoryManager : public vespalib::Printable,
- public MemoryManagerInterface
-{
- AllocationLogic::UP _logic;
- vespalib::Lock _typeLock;
- std::map<std::string, MemoryAllocationType::UP> _types;
-
-public:
- typedef std::unique_ptr<MemoryManager> UP;
-
- MemoryManager(AllocationLogic::UP);
- ~MemoryManager();
-
- void setMaximumMemoryUsage(uint64_t max) override;
- virtual void getState(MemoryState& state, bool resetMax = false);
-
- const MemoryAllocationType&registerAllocationType(const MemoryAllocationType& type) override;
- const MemoryAllocationType&getAllocationType(const std::string& name) const override;
-
- std::vector<const MemoryAllocationType*> getAllocationTypes() const override;
-
- MemoryToken::UP allocate(const MemoryAllocationType&, uint64_t min, uint64_t max,
- uint8_t p, ReduceMemoryUsageInterface* = 0) override;
-
- uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override;
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp
deleted file mode 100644
index 2351edd267c..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp
+++ /dev/null
@@ -1,225 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memorystate.h"
-#include <sstream>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".memory.state");
-
-namespace storage::framework::defaultimplementation {
-
-MemoryState::Entry::Entry()
- : _currentUsedSize(0),
- _totalUserCount(0),
- _currentUserCount(0),
- _wantedCount(0),
- _minimumCount(0),
- _deniedCount(0),
- _forcedBeyondMaximumCount(0)
-{
-}
-
-void
-MemoryState::Entry::operator+=(const Entry& other)
-{
- _currentUsedSize += other._currentUsedSize;
- _currentUserCount += other._currentUserCount;
- _totalUserCount += other._totalUserCount;
- _wantedCount += other._wantedCount;
- _minimumCount += other._minimumCount;
- _deniedCount += other._deniedCount;
- _forcedBeyondMaximumCount += other._forcedBeyondMaximumCount;
-}
-
-MemoryState::SnapShot&
-MemoryState::SnapShot::operator+=(const MemoryState::SnapShot& other)
-{
- for (AllocationMap::const_iterator it = other._allocations.begin();
- it != other._allocations.end(); ++it)
- {
- PriorityMap& map(_allocations[it->first]);
- for (PriorityMap::const_iterator it2 = it->second.begin();
- it2 != it->second.end(); ++it2)
- {
- Entry& entry(map[it2->first]);
- entry += it2->second;
- }
- }
- return *this;
-}
-
-uint64_t
-MemoryState::SnapShot::getUserCount() const
-{
- uint64_t count = 0;
- for (AllocationMap::const_iterator it = _allocations.begin();
- it != _allocations.end(); ++it)
- {
- for (PriorityMap::const_iterator it2 = it->second.begin();
- it2 != it->second.end(); ++it2)
- {
- count += it2->second._currentUserCount;
- }
- }
- return count;
-}
-
-MemoryState::MemoryState(Clock& clock, uint64_t maxMemory)
- : _clock(&clock),
- _maxMemory(maxMemory),
- _current(),
- _max(),
- _minJumpToUpdateMax(10 * 1024 * 1024)
-{
-}
-
-MemoryState::MemoryState(const MemoryState &) = default;
-MemoryState & MemoryState::operator = (const MemoryState &) = default;
-
-MemoryState::~MemoryState() {}
-
-void
-MemoryState::addToEntry(const MemoryAllocationType& type, uint64_t memory,
- uint8_t priority,
- AllocationResult result, bool forcedAllocation,
- uint64_t allocationCounts)
-{
- LOG(spam, "Allocating memory %s - %lu bytes at priority %u. "
- "Count %lu.",
- type.getName().c_str(), memory, priority, allocationCounts);
- PriorityMap& map(_current._allocations[&type]);
- Entry& e(map[priority]);
- e._currentUsedSize += memory;
- e._totalUserCount += allocationCounts;
- if (allocationCounts == 0) {
- // Resizes adding no more users still count as another total
- // allocation attempt.
- ++e._totalUserCount;
- }
- e._currentUserCount += allocationCounts;
- switch (result) {
- case GOT_MAX: ++e._wantedCount; break;
- case GOT_MIN: ++e._minimumCount; break;
- case DENIED: ++e._deniedCount; break;
- }
- if (forcedAllocation) ++e._forcedBeyondMaximumCount;
- _current._usedMemory += memory;
- if (!type.isCache()) {
- _current._usedWithoutCache += memory;
- }
- if (_current._usedWithoutCache
- > _max._usedWithoutCache + _minJumpToUpdateMax)
- {
- LOG(spam, "Updating max to current %lu bytes of memory used",
- _current._usedWithoutCache);
- _max = _current;
- _max._timeTaken = _clock->getTimeInSeconds();
- }
-}
-
-void
-MemoryState::removeFromEntry(const MemoryAllocationType& type, uint64_t memory,
- uint8_t priority,
- uint64_t allocationCounts)
-{
- LOG(spam, "Freeing memory %s - %lu bytes at priority %u. "
- "Count %lu.",
- type.getName().c_str(), memory, priority, allocationCounts);
- PriorityMap& map(_current._allocations[&type]);
- Entry& e(map[priority]);
- e._currentUsedSize -= memory;
- e._currentUserCount -= allocationCounts;
- _current._usedMemory -= memory;
- if (!type.isCache()) {
- _current._usedWithoutCache -= memory;
- }
-}
-
-void
-MemoryState::Entry::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- (void) verbose; (void) indent;
- std::ostringstream ost;
- ost << "Used(" << _currentUsedSize << " B / "
- << _currentUserCount << ") ";
- for (uint32_t i=ost.str().size(); i<20; ++i) {
- ost << " ";
- }
-
- out << ost.str()
- << "Stats(" << _totalUserCount
- << ", " << _wantedCount
- << ", " << _minimumCount
- << ", " << _deniedCount
- << ", " << _forcedBeyondMaximumCount << ")";
-}
-
-namespace {
- void printAllocations(std::ostream& out,
- const MemoryState::AllocationMap& map,
- const std::string& indent)
- {
- std::map<std::string, std::string> allocs;
-
- for (MemoryState::AllocationMap::const_iterator it = map.begin();
- it != map.end(); ++it)
- {
- for (MemoryState::PriorityMap::const_iterator it2
- = it->second.begin(); it2 != it->second.end(); ++it2)
- {
- std::ostringstream name;
- name << it->first->getName() << "("
- << static_cast<uint16_t>(it2->first) << "): ";
- for (uint32_t i=name.str().size(); i<25; ++i) {
- name << " ";
- }
-
- std::ostringstream tmp;
- it2->second.print(tmp, true, indent + " ");
-
- allocs[name.str()] = tmp.str();
- }
- }
-
- for (std::map<std::string, std::string>::const_iterator it
- = allocs.begin(); it != allocs.end(); ++it)
- {
- out << "\n" << indent << " " << it->first << it->second;
- }
- }
-}
-
-void
-MemoryState::SnapShot::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- out << "SnapShot(Used " << _usedMemory << ", w/o cache "
- << _usedWithoutCache;
- if (verbose) {
- out << ") {";
- if (_usedMemory > 0) {
- out << "\n" << indent << " Type(Pri): Used(Size/Allocs) "
- << "Stats(Allocs, Wanted, Min, Denied, Forced)";
- }
- printAllocations(out, _allocations, indent);
- out << "\n" << indent << "}";
- }
-}
-
-void
-MemoryState::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- bool maxSet = (_max._usedWithoutCache > _current._usedWithoutCache);
- out << "MemoryState(Max memory: " << _maxMemory << ") {"
- << "\n" << indent << " Current: ";
- _current.print(out, verbose, indent + " ");
- if (maxSet) {
- out << "\n" << indent << " Max: ";
- _max.print(out, verbose, indent + " ");
- }
- out << "\n" << indent << "}";
-}
-
-}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h
deleted file mode 100644
index 3d670d43844..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h
+++ /dev/null
@@ -1,139 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::MemoryState
- *
- * \brief Shows the state of current memory users
- *
- */
-
-#pragma once
-
-#include "memorymanager.h"
-#include <vespa/storageframework/generic/clock/clock.h>
-#include <vespa/vespalib/util/sync.h>
-
-namespace storage::framework::defaultimplementation {
-
-class MemoryState : public vespalib::Printable {
-public:
- struct Entry {
- // Total number of bytes allocated to this entry right now
- uint64_t _currentUsedSize;
- // Total number of allocations done on this entry
- uint64_t _totalUserCount;
- // Total number of allocations for this entry right now
- uint32_t _currentUserCount;
- // Amount of times this entry has gotten all the memory it wanted
- uint32_t _wantedCount;
- // Amount of times this entry has gotten less than all the memory
- // it wanted
- uint32_t _minimumCount;
- // Amount of times this entry has been denied getting memory
- uint32_t _deniedCount;
- // Amount of times this entry has forced memory allocations beyond
- // the maximum
- uint32_t _forcedBeyondMaximumCount;
-
- Entry();
-
- void print(std::ostream& out, bool verbose,
- const std::string& indent) const;
-
- /**
- * Set this instances counts to the counts from the other entry.
- */
- void transferCounts(const Entry& other);
-
- void operator+=(const Entry& other);
- };
-
- typedef std::map<uint8_t, Entry> PriorityMap;
- typedef std::map<const MemoryAllocationType*, PriorityMap> AllocationMap;
-
- /**
- * A snapshot contains data for either current or max seen data.
- * When a new maximum is seen, current is copied to max.
- */
- class SnapShot : public vespalib::Printable {
- friend class MemoryState;
-
- uint64_t _usedMemory;
- uint64_t _usedWithoutCache;
- SecondTime _timeTaken;
- AllocationMap _allocations;
-
- public:
- SnapShot() : vespalib::Printable() { clear(); }
- SnapShot(const SnapShot& o) : vespalib::Printable() { (*this) = o; }
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- void clear() {
- _usedMemory = 0;
- _usedWithoutCache = 0;
- _timeTaken.setTime(0);
- _allocations.clear();
- }
-
- SnapShot& operator=(const SnapShot& other) {
- _usedMemory = other._usedMemory;
- _usedWithoutCache = other._usedWithoutCache;
- _timeTaken = other._timeTaken;
- _allocations = other._allocations;
- return *this;
- }
-
- SnapShot& operator+=(const SnapShot& other);
-
- const AllocationMap& getAllocations() const { return _allocations; }
- uint64_t getUsedSize() const { return _usedMemory; }
- uint64_t getUsedSizeIgnoringCache() const { return _usedWithoutCache; }
- uint64_t getUserCount() const;
- };
-
-private:
- Clock* _clock;
- uint64_t _maxMemory;
- SnapShot _current;
- SnapShot _max;
- uint32_t _minJumpToUpdateMax;
-
-public:
- MemoryState(Clock& clock, uint64_t maxMemory);
- MemoryState(const MemoryState &);
- MemoryState & operator = (const MemoryState &);
- MemoryState(MemoryState &&) = default;
- MemoryState & operator = (MemoryState &&) = default;
- ~MemoryState();
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- void setMaximumMemoryUsage(uint64_t max) { _maxMemory = max; }
- void setMinJumpToUpdateMax(uint32_t bytes) { _minJumpToUpdateMax = bytes; }
-
- enum AllocationResult { GOT_MAX, GOT_MIN, DENIED };
- void addToEntry(const MemoryAllocationType& type, uint64_t memory,
- uint8_t priority,
- AllocationResult result, bool forcedAllocation = false,
- uint64_t allocationCounts = 1);
-
- void removeFromEntry(const MemoryAllocationType& type, uint64_t memory,
- uint8_t priority,
- uint64_t allocationCounts = 1);
- void resetMax() {
- _max = _current;
- _max._timeTaken = _clock->getTimeInSeconds();
- }
-
- const SnapShot& getCurrentSnapshot() const { return _current; }
- const SnapShot& getMaxSnapshot() const { return _max; }
-
-
- uint64_t getTotalSize() const { return _maxMemory; }
- uint64_t getFreeSize() const {
- return _maxMemory > _current._usedMemory
- ? _maxMemory - _current._usedMemory : 0;
- }
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp
deleted file mode 100644
index 16e03240a4f..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "nomemorymanager.h"
-#include <vespa/vespalib/util/exceptions.h>
-
-namespace storage::framework::defaultimplementation {
-
-NoMemoryManager::~NoMemoryManager() {}
-
-const MemoryAllocationType&
-NoMemoryManager::registerAllocationType(const MemoryAllocationType& type)
-{
- vespalib::LockGuard lock(_typeLock);
- _types[type.getName()] = MemoryAllocationType::UP(
- new MemoryAllocationType(type));
- return *_types[type.getName()];
-}
-
-const MemoryAllocationType&
-NoMemoryManager::getAllocationType(const std::string& name) const
-{
- vespalib::LockGuard lock(_typeLock);
- std::map<std::string, MemoryAllocationType::UP>::const_iterator it(
- _types.find(name));
- if (it == _types.end()) {
- throw vespalib::IllegalArgumentException(
- "Allocation type not found: " + name, VESPA_STRLOC);
- }
- return *it->second;
-}
-
-std::vector<const MemoryAllocationType*>
-NoMemoryManager::getAllocationTypes() const
-{
- vespalib::LockGuard lock(_typeLock);
- std::vector<const MemoryAllocationType*> types;
- for(std::map<std::string, MemoryAllocationType::UP>::const_iterator it
- = _types.begin(); it != _types.end(); ++it)
- {
- types.push_back(it->second.get());
- }
- return types;
-}
-
-}
-
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h
deleted file mode 100644
index 7f90b1360b8..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::NoMemoryManager
- *
- * \brief Memory manager that gives out max memory to everyone.
- *
- * Memory manager to use for testing and for apps not wanting to track memory.
- * This manager will merely give out max to everyone who asks and not even keep
- * track of anything.
- */
-
-#pragma once
-
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
-#include <vespa/vespalib/util/printable.h>
-#include <vespa/vespalib/util/sync.h>
-#include <map>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-class SimpleMemoryTokenImpl : public MemoryToken
-{
- uint64_t _allocated;
-
-public:
- SimpleMemoryTokenImpl(const SimpleMemoryTokenImpl &) = delete;
- SimpleMemoryTokenImpl & operator = (const SimpleMemoryTokenImpl &) = delete;
- SimpleMemoryTokenImpl(uint64_t allocated) : _allocated(allocated) {}
-
- uint64_t getSize() const override { return _allocated; }
- bool resize(uint64_t /* min */, uint64_t max) override { _allocated = max; return true; }
-};
-
-class NoMemoryManager : public MemoryManagerInterface
-{
- vespalib::Lock _typeLock;
- std::map<std::string, MemoryAllocationType::UP> _types;
-
-public:
- typedef std::unique_ptr<NoMemoryManager> UP;
-
- ~NoMemoryManager();
-
- void setMaximumMemoryUsage(uint64_t) override {}
- const MemoryAllocationType & registerAllocationType(const MemoryAllocationType& type) override;
- const MemoryAllocationType & getAllocationType(const std::string& name) const override;
-
- MemoryToken::UP allocate(const MemoryAllocationType&, uint64_t /* min */, uint64_t max,
- uint8_t /* priority */, ReduceMemoryUsageInterface* = 0) override
- {
- return SimpleMemoryTokenImpl::UP(new SimpleMemoryTokenImpl(max));
- }
- uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override {
- (void) priority;
- return std::numeric_limits<uint64_t>().max();
- }
-
- std::vector<const MemoryAllocationType*> getAllocationTypes() const override;
-};
-
-} // defaultimplementation
-} // framework
-} // storage
-
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp
deleted file mode 100644
index 51bb7ba5019..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "prioritymemorylogic.h"
-
-#include <vespa/log/log.h>
-LOG_SETUP(".memory.logic.priority");
-
-namespace storage::framework::defaultimplementation {
-
-PriorityMemoryLogic::PriorityMemoryLogic(Clock& c, uint64_t maxMem)
- : SimpleMemoryLogic(c, maxMem)
-{
- LOG(debug, "Setup priority memory logic with max memory of %lu bytes", maxMem);
-}
-
-float
-PriorityMemoryLogic::getNonCacheThreshold(uint8_t priority) const
-{
- return 0.6 + ((255 - priority) / 255.0) * 0.4;
-}
-
-void
-PriorityMemoryLogic::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- out << "PriorityMemoryLogic() : ";
- SimpleMemoryLogic::print(out, verbose, indent);
-}
-
-}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h
deleted file mode 100644
index 148c4f40aee..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-/**
- * \class storage::PriorityMemoryLogic
- *
- * \brief Priority logic deciding who should get memory and how much.
- *
- */
-
-#pragma once
-
-#include "simplememorylogic.h"
-#include <vespa/vespalib/util/sync.h>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-struct PriorityMemoryLogic : public SimpleMemoryLogic
-{
- PriorityMemoryLogic(Clock&, uint64_t maxMemory);
- float getNonCacheThreshold(uint8_t priority) const override;
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-};
-
-} // defaultimplementation
-} // framework
-} // storage
-
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp
deleted file mode 100644
index 76d6f990a00..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp
+++ /dev/null
@@ -1,232 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "simplememorylogic.h"
-#include <vespa/vespalib/util/exceptions.h>
-
-#include <vespa/log/bufferedlogger.h>
-LOG_SETUP(".memory.logic.simple");
-
-namespace storage::framework::defaultimplementation {
-
-SimpleMemoryLogic::SimpleMemoryLogic(Clock& c, uint64_t maxMemory)
- : _cacheThreshold(0.98),
- _nonCacheThreshold(0.8),
- _state(c, maxMemory),
- _reducers()
-{
- LOG(debug, "Setup simple memory logic with max memory of %" PRIu64 " bytes", maxMemory);
-}
-
-SimpleMemoryLogic::~SimpleMemoryLogic()
-{
-}
-
-void
-SimpleMemoryLogic::setMaximumMemoryUsage(uint64_t max)
-{
- vespalib::LockGuard lock(_stateLock);
- _state.setMaximumMemoryUsage(max);
-}
-
-void
-SimpleMemoryLogic::getState(MemoryState& state, bool resetMax) {
- vespalib::LockGuard lock(_stateLock);
- state = _state;
- if (resetMax) _state.resetMax();
-}
-
-MemoryToken::UP
-SimpleMemoryLogic::allocate(const MemoryAllocationType& type,
- uint8_t priority,
- ReduceMemoryUsageInterface* reducer)
-{
- MemoryTokenImpl::UP token(
- new MemoryTokenImpl(*this, type, 0, priority, reducer));
- if (reducer != 0) {
- vespalib::LockGuard lock(_stateLock);
- _reducers.push_back(Reducer(*token, *reducer));
- }
- return std::move(token);
-}
-
-bool
-SimpleMemoryLogic::resize(MemoryToken& tok, uint64_t min, uint64_t max,
- uint32_t allocationCounts)
-{
- vespalib::LockGuard lock(_stateLock);
- MemoryTokenImpl& token(static_cast<MemoryTokenImpl&>(tok));
- LOG(spam, "Attempting to resize %s to size in the range %" PRIu64 " to "
- "%" PRIu64 ".", token.toString().c_str(), min, max);
- if (token.getSize() > max) { // Always safe to reduce size
- handleReduction(token, max, allocationCounts);
- return true;
- }
- // If not reducing size, calculate relative min/max values.
- uint64_t relMin = (min > token.getSize() ? min - token.getSize() : 0);
- uint64_t relMax = max - token.getSize();
- return resizeRelative(token, relMin, relMax, allocationCounts);
-}
-
-void
-SimpleMemoryLogic::handleReduction(MemoryTokenImpl& token, uint64_t max,
- uint32_t allocationCounts)
-{
- LOG(spam, "Reduzing size of token by %" PRIu64 ".",
- token.getSize() - max);
- _state.removeFromEntry(token.getType(), token.getSize() - max,
- token.getPriority(), allocationCounts);
- setTokenSize(token, max);
-}
-
-bool
-SimpleMemoryLogic::handleCacheMemoryRequest(
- MemoryTokenImpl& token, uint64_t min, uint64_t max,
- uint32_t allocationCounts)
-{
- uint64_t usedSize(_state.getCurrentSnapshot().getUsedSize());
- uint64_t thresholdSize = uint64_t(getCacheThreshold()
- * _state.getTotalSize());
- uint64_t toAllocate(thresholdSize > usedSize
- ? std::min(thresholdSize - usedSize, max)
- : 0);
- bool forced = false;
- if (token.getType().isAllocationsForced() && toAllocate < min) {
- toAllocate = min;
- forced = true;
- }
- if (toAllocate < min) {
- LOG(spam, "We cannot give more memory to cache without going above "
- "cache threshold (%" PRIu64 " B)", thresholdSize);
- _state.addToEntry(token.getType(), 0, token.getPriority(),
- MemoryState::DENIED, false, allocationCounts);
- return false;
- }
- LOG(spam, "Giving %" PRIu64 " bytes of memory to cache. (Cache threshold "
- "is %" PRIu64 ", used size is %" PRIu64 ", %" PRIu64 " bytes were "
- "always allocated to the token and it wanted memory between %"
- PRIu64 " and %" PRIu64 ".",
- toAllocate, thresholdSize, usedSize, token.getSize(), min, max);
- _state.addToEntry(token.getType(), toAllocate, token.getPriority(),
- static_cast<uint64_t>(toAllocate) >= max
- ? MemoryState::GOT_MAX : MemoryState::GOT_MIN,
- forced, allocationCounts);
- setTokenSize(token, token.getSize() + toAllocate);
- return true;
-}
-
-uint64_t
-SimpleMemoryLogic::getMemorySizeFreeForPriority(uint8_t priority) const
-{
- uint64_t usedSize(_state.getCurrentSnapshot().getUsedSizeIgnoringCache());
- uint64_t thresholdSize = uint64_t(getNonCacheThreshold(priority)
- * _state.getTotalSize());
- return (usedSize >= thresholdSize ? 0 : thresholdSize - usedSize);
-}
-
-bool
-SimpleMemoryLogic::resizeRelative(
- MemoryTokenImpl& token, uint64_t min, uint64_t max,
- uint32_t allocationCounts)
-{
- LOG(spam, "Relative resize change. Need another %zu-%zu byte of memory.",
- min, max);
- // If requester is cache, use cache threshold
- if (token.getType().isCache()) {
- return handleCacheMemoryRequest(token, min, max, allocationCounts);
- }
- // If we get here, requester is not cache.
- uint64_t usedSize(_state.getCurrentSnapshot().getUsedSizeIgnoringCache());
- uint64_t thresholdSize = uint64_t(getNonCacheThreshold(token.getPriority())
- * _state.getTotalSize());
- uint64_t toAllocate = 0;
- if (thresholdSize > usedSize) {
- toAllocate = std::min(max, thresholdSize - usedSize);
- }
- if (toAllocate < min) toAllocate = min;
- bool forced = false;
- if (usedSize + toAllocate > _state.getTotalSize()) {
- if (token.getType().isAllocationsForced()) {
- forced = true;
- } else {
- LOG(spam, "We cannot give more memory without going beyond max");
- _state.addToEntry(token.getType(), 0, token.getPriority(),
- MemoryState::DENIED, false, allocationCounts);
- return false;
- }
- }
- // External load should not fill up too much
- if (usedSize + toAllocate > thresholdSize
- && token.getType().isExternalLoad()
- && !token.getType().isAllocationsForced())
- {
- LOG(spam, "Not giving external load memory beyond threshold.");
- _state.addToEntry(token.getType(), 0, token.getPriority(),
- MemoryState::DENIED, false, allocationCounts);
- return false;
- }
- // If this puts us above max with cache, remove some cache.
- if (_state.getCurrentSnapshot().getUsedSize() + toAllocate
- > _state.getTotalSize())
- {
- uint64_t needed(_state.getCurrentSnapshot().getUsedSize()
- + toAllocate - _state.getTotalSize());
- for (uint32_t i=0; i<_reducers.size(); ++i) {
- MemoryTokenImpl& rtoken(*_reducers[i]._token);
- uint64_t reduceBy(std::min(needed, rtoken.getSize()));
- uint64_t reduced(_reducers[i]._reducer->reduceMemoryConsumption(
- rtoken, reduceBy));
- _state.removeFromEntry(rtoken.getType(), reduced,
- rtoken.getPriority(), 0);
- setTokenSize(rtoken, rtoken.getSize() - reduced);
- needed -= reduceBy;
- if (needed == 0) break;
- if (reduced < reduceBy) {
- LOG(debug, "Reducer refused to free the full %" PRIu64 " bytes "
- "requested. %" PRIu64 " bytes reduced in token %s.",
- reduceBy, reduced, rtoken.toString().c_str());
- }
- }
- }
- if (_state.getCurrentSnapshot().getUsedSize() + toAllocate
- > _state.getTotalSize())
- {
- LOGBP(debug, "Failed to free enough memory from cache. This puts us "
- "above max memory.");
- }
- LOG(spam, "Giving %" PRIu64 " bytes of memory", toAllocate);
- _state.addToEntry(token.getType(), toAllocate, token.getPriority(),
- static_cast<uint64_t>(toAllocate) >= max
- ? MemoryState::GOT_MAX : MemoryState::GOT_MIN,
- forced, allocationCounts);
- setTokenSize(token, token.getSize() + toAllocate);
- return true;
-}
-
-void
-SimpleMemoryLogic::freeToken(MemoryTokenImpl& token)
-{
- vespalib::LockGuard lock(_stateLock);
- _state.removeFromEntry(token.getType(), token.getSize(),
- token.getPriority(), token.getAllocationCount());
- if (token.getReducer() != 0) {
- std::vector<Reducer> reducers;
- reducers.reserve(_reducers.size() - 1);
- for (uint32_t i=0; i<_reducers.size(); ++i) {
- if (_reducers[i]._token != &token) reducers.push_back(_reducers[i]);
- }
- assert(reducers.size() + 1 == _reducers.size());
- reducers.swap(_reducers);
- }
-}
-
-void
-SimpleMemoryLogic::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- out << "SimpleMemoryLogic() {\n"
- << indent << " ";
- vespalib::LockGuard lock(_stateLock);
- _state.print(out, verbose, indent + " ");
-}
-
-}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h
deleted file mode 100644
index c394db40b29..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-/**
- * \class storage::SimpleMemoryLogic
- *
- * \brief Simple logic deciding who should get memory and how much.
- *
- * There is a cache threshold. By default 98%. Cache will always get memory up
- * till this fillrate.
- *
- * There is a non-cache threshold. Non-cache memory requesters will get maximum
- * memory until threshold is reached. If getting maximum memory would go beyond
- * the non-cache threshold, the requester will get enough memory to hit the
- * threshold (if more than minimum), or get the minimum memory asked for, if
- * that doesn't put usage above 100%.
- *
- * Usage above 100% is attempted avoided by freeing cache memory. If failing to
- * free enough memory, request will fail, or minimum will be get if allocation
- * is forced such that it cannot fail. In such a case, usage may go beyond 100%.
- */
-
-#pragma once
-
-#include "memorymanager.h"
-#include "memorystate.h"
-#include <vespa/vespalib/util/sync.h>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-class SimpleMemoryLogic : public AllocationLogic
-{
- float _cacheThreshold;
- float _nonCacheThreshold;
- vespalib::Lock _stateLock;
- MemoryState _state;
- struct Reducer {
- MemoryTokenImpl* _token;
- ReduceMemoryUsageInterface* _reducer;
-
- Reducer() : _token(0), _reducer(0) {}
- Reducer(MemoryTokenImpl& t,
- ReduceMemoryUsageInterface& r)
- : _token(&t), _reducer(&r) {}
- };
- std::vector<Reducer> _reducers;
-
-protected:
- float getCacheThreshold() { return _cacheThreshold; }
-
- // Priority memory logic can override this to set a threshold based on
- // priority
- virtual float getNonCacheThreshold(uint8_t priority) const
- { (void) priority; return _nonCacheThreshold; }
-
-public:
- typedef std::unique_ptr<SimpleMemoryLogic> UP;
-
- SimpleMemoryLogic(Clock&, uint64_t maxMemory);
-
- ~SimpleMemoryLogic();
-
- SimpleMemoryLogic& setMinJumpToUpdateMax(uint32_t bytes) {
- _state.setMinJumpToUpdateMax(bytes);
- return *this;
- }
-
- void setMaximumMemoryUsage(uint64_t max) override;
-
- void setCacheThreshold(float limit) { _cacheThreshold = limit; }
- void setNonCacheThreshold(float limit) { _nonCacheThreshold = limit; }
-
- MemoryState& getState() { return _state; } // Not threadsafe. Unit testing.
- void getState(MemoryState& state, bool resetMax) override;
-
- MemoryToken::UP allocate(const MemoryAllocationType&, uint8_t priority,
- ReduceMemoryUsageInterface* = 0) override;
- bool resize(MemoryToken& token, uint64_t min, uint64_t max, uint32_t allocationCounts) override;
-
- void freeToken(MemoryTokenImpl& token) override;
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- virtual uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override;
-
-private:
- void handleReduction(MemoryTokenImpl&, uint64_t size,
- uint32_t allocationCounts);
- bool resizeRelative(MemoryTokenImpl&, uint64_t min, uint64_t max,
- uint32_t allocationCounts);
- bool handleCacheMemoryRequest(MemoryTokenImpl&, uint64_t min, uint64_t max,
- uint32_t allocationCounts);
-};
-
-} // defaultimplementation
-} // framework
-} // storage
-
diff --git a/storageframework/src/vespa/storageframework/generic/CMakeLists.txt b/storageframework/src/vespa/storageframework/generic/CMakeLists.txt
index cf225f8d5aa..5df9963304d 100644
--- a/storageframework/src/vespa/storageframework/generic/CMakeLists.txt
+++ b/storageframework/src/vespa/storageframework/generic/CMakeLists.txt
@@ -4,7 +4,6 @@ vespa_add_library(storageframework_generic
$<TARGET_OBJECTS:storageframework_component>
$<TARGET_OBJECTS:storageframework_status>
$<TARGET_OBJECTS:storageframework_thread>
- $<TARGET_OBJECTS:storageframework_memory>
$<TARGET_OBJECTS:storageframework_clock>
INSTALL lib64
DEPENDS
diff --git a/storageframework/src/vespa/storageframework/generic/memory/.gitignore b/storageframework/src/vespa/storageframework/generic/memory/.gitignore
deleted file mode 100644
index 7e7c0fe7fae..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/.depend
-/Makefile
diff --git a/storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt b/storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt
deleted file mode 100644
index 77aa14e4ca2..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(storageframework_memory OBJECT
- SOURCES
- memorytoken.cpp
- DEPENDS
-)
diff --git a/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h b/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h
deleted file mode 100644
index e91b7946454..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::framework::MemoryAllocationType
- * \ingroup memory
- *
- * \brief Allocation types used to differ between memory manager clients.
- *
- * The different memory manager clients have different properties. It is
- * important for the memory manager to distinguish between different users in
- * order to know how to prioritize memory, and also in order to create good
- * reports on memory usage.
- *
- * An allocation type holds metadata for a memory manager client, including a
- * name for the type and various properties that may affect how much memory
- * such a client will get, whether it always gets some, etc.
- */
-
-#pragma once
-
-#include <string>
-#include <memory>
-
-namespace storage::framework {
-
-struct MemoryAllocationType {
- using UP = std::unique_ptr<MemoryAllocationType>;
-
- enum Flags {
- NONE = 0x00,
- FORCE_ALLOCATE = 0x01,
- EXTERNAL_LOAD = 0x02,
- CACHE = 0x04
- };
-
- MemoryAllocationType()
- : _flags(NONE), _name("") {};
-
- MemoryAllocationType(const std::string& name, uint32_t flags = NONE)
- : _flags(flags), _name(name) {}
-
- const std::string& getName() const { return _name; }
- bool isAllocationsForced() const { return (_flags & FORCE_ALLOCATE); }
- bool isExternalLoad() const { return (_flags & EXTERNAL_LOAD); }
- bool isCache() const { return (_flags & CACHE); }
-
-private:
- uint32_t _flags;
- std::string _name;
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h b/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h
deleted file mode 100644
index a97038c2aab..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::framework::MemoryManagerInterface
- * \ingroup memory
- *
- * \brief Interface with functions clients need in order to use a memory manager
- *
- * This interface exist so clients can use a memory manager without actually
- * depending on the implementation of it.
- */
-
-#pragma once
-
-#include "memoryallocationtype.h"
-#include "memorytoken.h"
-#include "reducememoryusageinterface.h"
-#include <vector>
-
-namespace storage::framework {
-
-struct MemoryManagerInterface
-{
- typedef std::unique_ptr<MemoryManagerInterface> UP;
-
- virtual ~MemoryManagerInterface() {}
-
- virtual void setMaximumMemoryUsage(uint64_t max) = 0;
-
- /**
- * Registers the given allocation type by copying it, and returning
- * a reference to the copied object.
- */
- virtual const MemoryAllocationType&
- registerAllocationType(const MemoryAllocationType& type) = 0;
-
- /** Throws exception if failing to find type. */
- virtual const MemoryAllocationType&
- getAllocationType(const std::string& name) const = 0;
-
- /** Get an overview of all registration types. */
- virtual std::vector<const MemoryAllocationType*>
- getAllocationTypes() const = 0;
-
- /**
- * Decide how much to allocate for this request. Should be between min
- * and max, unless it's of a type that can be denied (such as external
- * requests), in which case we can also deny allocation by returning a null
- * token.
- */
- virtual MemoryToken::UP allocate(
- const MemoryAllocationType&,
- uint64_t min,
- uint64_t max,
- uint8_t priority,
- ReduceMemoryUsageInterface* = 0) = 0;
-
- /**
- * Utility function to see how much memory is available.
- */
- virtual uint64_t getMemorySizeFreeForPriority(uint8_t priority) const = 0;
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp b/storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp
deleted file mode 100644
index 3950810a010..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memorytoken.h"
-
-namespace storage {
-namespace framework {
-
-MemoryToken::~MemoryToken()
-{
-}
-
-} // framework
-} // storage
diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h b/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h
deleted file mode 100644
index cd389d64549..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::framework::MemoryToken
- * \ingroup memory
- *
- * \brief Token to keep by client for current allocations.
- *
- * This class is a token a memory manager client will get from the memory
- * manager when getting memory. It can be used to know how much you currently
- * have allocated, and through it you can request more or less memory.
- */
-
-#pragma once
-
-#include <memory>
-
-namespace storage::framework {
-
-class MemoryToken {
-protected:
-public:
- typedef std::unique_ptr<MemoryToken> UP;
- virtual ~MemoryToken();
-
- virtual uint64_t getSize() const = 0;
- virtual bool resize(uint64_t min, uint64_t max) = 0;
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h b/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h
deleted file mode 100644
index cd50d33015c..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::framework::ReduceMemoryUsageInterface
- * \ingroup memory
- *
- * \brief The manager can take memory back when needed using this interface.
- *
- * Some memory users, typically caches, wants to use all available memory. But
- * to let them use all available memory, it must also be easy to take memory
- * back when needed for something else. An implementation of this interface can
- * be given on memory allocations to give the memory manager the ability to take
- * memory back when needed.
- */
-
-#pragma once
-
-namespace storage::framework {
-
-struct ReduceMemoryUsageInterface
-{
- virtual ~ReduceMemoryUsageInterface() {}
-
- /**
- * This callback is called when the memory manager want to reduce the usage
- * of the given memory token. Actual memory to be released should be
- * released in this function. The token itself will be adjusted by the
- * memory manager though. The memory manager may keep a lock through this
- * call, so no memory manager calls should be made inside this callback.
- *
- * It is recommended that you actually release at least as many bytes as
- * requested. Though currently it is allowed to reduce less or refuse, but
- * this might mean that some higher priority task does not get the memory it
- * needs.
- *
- * @param reduceBy Always in the range 0 < reduceBy <= token.size()
- * @return The amount of memory no longer used.
- */
- virtual uint64_t reduceMemoryConsumption(const MemoryToken&, uint64_t reduceBy) = 0;
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/storageframework.h b/storageframework/src/vespa/storageframework/storageframework.h
index f9847d5fd12..4ab36fa099f 100644
--- a/storageframework/src/vespa/storageframework/storageframework.h
+++ b/storageframework/src/vespa/storageframework/storageframework.h
@@ -7,7 +7,6 @@
#include <vespa/storageframework/generic/clock/clock.h>
#include <vespa/storageframework/generic/clock/timer.h>
#include <vespa/storageframework/generic/component/component.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/generic/metric/metricupdatehook.h>
#include <vespa/storageframework/generic/status/htmlstatusreporter.h>
#include <vespa/storageframework/generic/status/statusreportermap.h>