summaryrefslogtreecommitdiffstats
path: root/storageframework
diff options
context:
space:
mode:
authorTor Brede Vekterli <vekterli@yahoo-inc.com>2017-12-04 11:31:29 +0000
committerTor Brede Vekterli <vekterli@yahoo-inc.com>2017-12-04 13:06:46 +0000
commit952a877611d657cfa0166b14699c8731b18f7587 (patch)
treec1b4330b9dc512c3ffbf845e626857ddcd801982 /storageframework
parentd164fbb93e277ef23ab610320a7cf8556e3c036e (diff)
Remove memory manager component from content layer
We already have resource utilization tracking in both MessageBus and the search core. The memory manager has never been auto-scaled based on the hardware present and adds a _lot_ of complexity without having any known instances where it has actually saved the day. Removing it also removes a mutex on the message hot path. If we need such functionality in the future, should design a lock-free solution. Cleanup
Diffstat (limited to 'storageframework')
-rw-r--r--storageframework/CMakeLists.txt3
-rw-r--r--storageframework/src/tests/CMakeLists.txt1
-rw-r--r--storageframework/src/tests/memory/.gitignore2
-rw-r--r--storageframework/src/tests/memory/CMakeLists.txt8
-rw-r--r--storageframework/src/tests/memory/memorymanagertest.cpp397
-rw-r--r--storageframework/src/tests/memory/memorystatetest.cpp172
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt1
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp10
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h1
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp6
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h3
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore2
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt10
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h48
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp157
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h156
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp225
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h139
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp46
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h66
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp30
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h29
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp232
-rw-r--r--storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h98
-rw-r--r--storageframework/src/vespa/storageframework/generic/CMakeLists.txt1
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/.gitignore2
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt6
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h51
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h63
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp13
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/memorytoken.h29
-rw-r--r--storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h41
-rw-r--r--storageframework/src/vespa/storageframework/storageframework.h1
33 files changed, 1 insertions, 2048 deletions
diff --git a/storageframework/CMakeLists.txt b/storageframework/CMakeLists.txt
index 5a4b2e02372..a3414a8b05c 100644
--- a/storageframework/CMakeLists.txt
+++ b/storageframework/CMakeLists.txt
@@ -11,12 +11,10 @@ vespa_define_module(
src/vespa/storageframework/defaultimplementation
src/vespa/storageframework/defaultimplementation/clock
src/vespa/storageframework/defaultimplementation/component
- src/vespa/storageframework/defaultimplementation/memory
src/vespa/storageframework/defaultimplementation/thread
src/vespa/storageframework/generic
src/vespa/storageframework/generic/clock
src/vespa/storageframework/generic/component
- src/vespa/storageframework/generic/memory
src/vespa/storageframework/generic/metric
src/vespa/storageframework/generic/status
src/vespa/storageframework/generic/thread
@@ -28,7 +26,6 @@ vespa_define_module(
TESTS
src/tests
src/tests/clock
- src/tests/memory
src/tests/status
src/tests/thread
)
diff --git a/storageframework/src/tests/CMakeLists.txt b/storageframework/src/tests/CMakeLists.txt
index eb6d45330e8..0d604d64f74 100644
--- a/storageframework/src/tests/CMakeLists.txt
+++ b/storageframework/src/tests/CMakeLists.txt
@@ -5,7 +5,6 @@ vespa_add_executable(storageframework_testrunner_app TEST
DEPENDS
storageframework_testclock
storageframework_teststatus
- storageframework_testmemory
storageframework_testthread
)
diff --git a/storageframework/src/tests/memory/.gitignore b/storageframework/src/tests/memory/.gitignore
deleted file mode 100644
index 7e7c0fe7fae..00000000000
--- a/storageframework/src/tests/memory/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/.depend
-/Makefile
diff --git a/storageframework/src/tests/memory/CMakeLists.txt b/storageframework/src/tests/memory/CMakeLists.txt
deleted file mode 100644
index 591a78f1046..00000000000
--- a/storageframework/src/tests/memory/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(storageframework_testmemory
- SOURCES
- memorymanagertest.cpp
- memorystatetest.cpp
- DEPENDS
- storageframework
-)
diff --git a/storageframework/src/tests/memory/memorymanagertest.cpp b/storageframework/src/tests/memory/memorymanagertest.cpp
deleted file mode 100644
index 5424b8dea82..00000000000
--- a/storageframework/src/tests/memory/memorymanagertest.cpp
+++ /dev/null
@@ -1,397 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
-#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h>
-#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h>
-#include <vespa/vdstestlib/cppunit/macros.h>
-#include <vespa/vespalib/util/document_runnable.h>
-#include <vespa/vespalib/util/random.h>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-struct MemoryManagerTest : public CppUnit::TestFixture
-{
- void testBasics();
- void testCacheAllocation();
- void testStress();
-
- CPPUNIT_TEST_SUITE(MemoryManagerTest);
- CPPUNIT_TEST(testBasics);
- CPPUNIT_TEST(testCacheAllocation);
- CPPUNIT_TEST(testStress);
- CPPUNIT_TEST_SUITE_END();
-};
-
-CPPUNIT_TEST_SUITE_REGISTRATION(MemoryManagerTest);
-
-void
-MemoryManagerTest::testBasics()
-{
- uint64_t maxMemory = 1000;
- RealClock clock;
- SimpleMemoryLogic* logic = new SimpleMemoryLogic(clock, maxMemory);
- AllocationLogic::UP allLogic(std::move(logic));
- MemoryManager manager(std::move(allLogic));
-
- const MemoryAllocationType& putAlloc(manager.registerAllocationType(
- MemoryAllocationType("put", MemoryAllocationType::EXTERNAL_LOAD)));
- const MemoryAllocationType& getAlloc(manager.registerAllocationType(
- MemoryAllocationType("get", MemoryAllocationType::EXTERNAL_LOAD)));
- const MemoryAllocationType& bufAlloc(manager.registerAllocationType(
- MemoryAllocationType("buffer")));
- const MemoryAllocationType& cacheAlloc(manager.registerAllocationType(
- MemoryAllocationType("cache", MemoryAllocationType::CACHE)));
- const MemoryState& state(logic->getState());
- const MemoryState::SnapShot& current(state.getCurrentSnapshot());
- // Basics
- {
- // * Getting a token, and release it back with correct behavior
- framework::MemoryToken::UP put = manager.allocate(putAlloc,
- 0, 100, 80);
- CPPUNIT_ASSERT(put.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(100), put->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(100), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(900), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
-
- // * Do the same while not being empty. Different type.
- framework::MemoryToken::UP get = manager.allocate(getAlloc,
- 30, 200, 50);
- CPPUNIT_ASSERT(get.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(200), get->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(300), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(700), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
-
- // * Do the same while not being empty. Same type.
- framework::MemoryToken::UP get2 = manager.allocate(
- getAlloc,
- 70,
- 150,
- 60);
-
- CPPUNIT_ASSERT(get2.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(150), get2->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(450), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(550), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
-
- // Non-external load
- // * Getting minimum when going beyond 80% full
- {
- framework::MemoryToken::UP filler = manager.allocate(putAlloc,
- 795, 795, 90);
- framework::MemoryToken::UP resize = manager.allocate(
- bufAlloc, 10, 90, 80);
- CPPUNIT_ASSERT(resize.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(10), resize->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(805), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(195), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
-
- // Non-external load
- // * Getting up to threshold if hitting it
- {
- framework::MemoryToken::UP filler = manager.allocate(putAlloc,
- 750, 750, 90);
- framework::MemoryToken::UP resize = manager.allocate(
- bufAlloc, 10, 90, 80);
- CPPUNIT_ASSERT(resize.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(50), resize->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(800), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(200), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
-
- // External load
- {
- // * Stopped when going beyond 80% full
- framework::MemoryToken::UP filler = manager.allocate(putAlloc,
- 795, 795, 90);
- framework::MemoryToken::UP put = manager.allocate(putAlloc,
- 10, 100, 80);
- CPPUNIT_ASSERT(put.get() == 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(795), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(205), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
-
- // External load
- {
- // * Getting up to threshold if hitting it
- framework::MemoryToken::UP filler = manager.allocate(putAlloc,
- 750, 750, 90);
- framework::MemoryToken::UP put = manager.allocate(putAlloc,
- 10, 100, 80);
- CPPUNIT_ASSERT(put.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(50), put->getSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(800), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(200), state.getFreeSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(1000), state.getTotalSize());
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
-
- // Test caching..
- {
- // Cache paradigm:
- // Allocate a token taking up no space at all.
- // Give it to your ReduceMemoryUsageInterface implementation.
- // Run resize on your token in that implementation to get memory and
- // return memory. That way locking should be easy when needed.
- struct ReduceI : public framework::ReduceMemoryUsageInterface {
- framework::MemoryToken::UP _token;
-
- uint64_t reduceMemoryConsumption(const MemoryToken& token, uint64_t reduceBy) override {
- assert(&token == _token.get());
- (void) &token;
- assert(_token->getSize() >= reduceBy);
- return reduceBy;
- }
- };
- ReduceI reducer;
- framework::MemoryToken::UP cache = manager.allocate(cacheAlloc,
- 0, 0, 0, &reducer);
- CPPUNIT_ASSERT(cache.get() != 0);
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), cache->getSize());
- reducer._token = std::move(cache);
- for (uint32_t i=1; i<=50; ++i) {
- bool success = reducer._token->resize(i * 10, i * 10);
- CPPUNIT_ASSERT_EQUAL(true, success);
- }
- CPPUNIT_ASSERT_EQUAL(uint64_t(500), reducer._token->getSize());
-
- // * Ordered to free space
- framework::MemoryToken::UP put = manager.allocate(putAlloc,
- 600, 600, 80);
- CPPUNIT_ASSERT_EQUAL_MSG(manager.toString(),
- uint64_t(400), reducer._token->getSize());
- CPPUNIT_ASSERT_EQUAL_MSG(manager.toString(),
- uint64_t(600), put->getSize());
- }
- CPPUNIT_ASSERT_EQUAL_MSG(state.toString(true),
- uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL_MSG(state.toString(true),
- uint64_t(0), current.getUserCount());
-
- // Test merge and tracking of allocation counts with merge, by doing
- // operations with tokens and see that user count and used size
- // correctly go back to zero.
- {
- framework::MemoryToken::UP tok1(
- manager.allocate(putAlloc, 5, 5, 40));
- framework::MemoryToken::UP tok2(
- manager.allocate(putAlloc, 10, 10, 40));
- framework::MemoryToken::UP tok3(
- manager.allocate(putAlloc, 20, 20, 40));
- framework::MemoryToken::UP tok4(
- manager.allocate(putAlloc, 40, 40, 40));
- framework::MemoryToken::UP tok5(
- manager.allocate(putAlloc, 80, 80, 40));
- framework::MemoryToken::UP tok6(
- manager.allocate(putAlloc, 1, 1, 40));
- framework::MemoryToken::UP tok7(
- manager.allocate(putAlloc, 3, 3, 40));
- }
-}
-
-void
-MemoryManagerTest::testCacheAllocation()
-{
- uint64_t maxMemory = 3000;
-
- RealClock clock;
- SimpleMemoryLogic::UP logic(new PriorityMemoryLogic(clock, maxMemory));
- logic->setCacheThreshold(1.0);
-
- AllocationLogic::UP allLogic(std::move(logic));
- MemoryManager manager(std::move(allLogic));
-
- const MemoryAllocationType& putAlloc(manager.registerAllocationType(
- MemoryAllocationType("put", MemoryAllocationType::EXTERNAL_LOAD)));
- const MemoryAllocationType& cacheAlloc(manager.registerAllocationType(
- MemoryAllocationType("cache", MemoryAllocationType::CACHE)));
-
- framework::MemoryToken::UP token =
- manager.allocate(putAlloc,
- 50,
- 50,
- 127);
-
- CPPUNIT_ASSERT_EQUAL(50, (int)token->getSize());
-
- framework::MemoryToken::UP token2 =
- manager.allocate(cacheAlloc,
- 1000,
- 2000,
- 127);
-
- CPPUNIT_ASSERT_EQUAL(2000, (int)token2->getSize());
-
- token2->resize(2000, 3000);
-
- CPPUNIT_ASSERT_EQUAL(2950, (int)token2->getSize());
-}
-
-namespace {
-struct MemoryManagerLoadGiver : public document::Runnable,
- public ReduceMemoryUsageInterface
-{
- MemoryManager& _manager;
- const framework::MemoryAllocationType& _type;
- uint8_t _priority;
- uint32_t _minMem;
- uint32_t _maxMem;
- uint32_t _failed;
- uint32_t _ok;
- uint32_t _reduced;
- using MemoryTokenUP = std::unique_ptr<MemoryToken>;
- std::vector<MemoryTokenUP> _tokens;
- vespalib::Lock _cacheLock;
-
- MemoryManagerLoadGiver(
- MemoryManager& manager,
- const framework::MemoryAllocationType& type,
- uint8_t priority,
- uint32_t minMem,
- uint32_t maxMem,
- uint32_t tokensToKeep)
- : _manager(manager),
- _type(type),
- _priority(priority),
- _minMem(minMem),
- _maxMem(maxMem),
- _failed(0),
- _ok(0),
- _reduced(0),
- _tokens(tokensToKeep)
- {
- }
-
- uint64_t reduceMemoryConsumption(const MemoryToken&, uint64_t reduceBy) override {
- ++_reduced;
- return reduceBy;
- }
-
- void run() override {
- ReduceMemoryUsageInterface* reducer = 0;
- if (_type.isCache()) reducer = this;
- vespalib::RandomGen randomizer;
- while (running()) {
- vespalib::Lock lock(_cacheLock);
- framework::MemoryToken::UP token = _manager.allocate(
- _type, _minMem, _maxMem, _priority, reducer);
- if (token.get() == 0) {
- ++_failed;
- } else {
- ++_ok;
- }
- uint32_t index = randomizer.nextUint32(0, _tokens.size() - 1);
- _tokens[index] = MemoryTokenUP(token.release());
- }
- }
-};
-}
-
-void
-MemoryManagerTest::testStress()
-{
- uint64_t stressTimeMS = 1 * 1000;
- uint64_t maxMemory = 1 * 1024 * 1024;
- RealClock clock;
- AllocationLogic::UP logic(new PriorityMemoryLogic(clock, maxMemory));
- MemoryManager manager(std::move(logic));
-
- FastOS_ThreadPool pool(128 * 1024);
- std::vector<MemoryManagerLoadGiver*> loadGivers;
- for (uint32_t type = 0; type < 5; ++type) {
- const MemoryAllocationType* allocType = 0;
- uint32_t min = 1000, max = 5000;
- if (type == 0) {
- allocType = &manager.registerAllocationType(MemoryAllocationType(
- "default"));
- } else if (type == 1) {
- allocType = &manager.registerAllocationType(MemoryAllocationType(
- "external", MemoryAllocationType::EXTERNAL_LOAD));
- } else if (type == 2) {
- allocType = &manager.registerAllocationType(MemoryAllocationType(
- "forced", MemoryAllocationType::FORCE_ALLOCATE));
- } else if (type == 3) {
- allocType = &manager.registerAllocationType(MemoryAllocationType(
- "forcedExternal", MemoryAllocationType::FORCE_ALLOCATE
- | MemoryAllocationType::EXTERNAL_LOAD));
- } else if (type == 4) {
- allocType = &manager.registerAllocationType(MemoryAllocationType(
- "cache", MemoryAllocationType::CACHE));
- max = 30000;
- }
- for (int priority = 0; priority < 256; priority += 8) {
- loadGivers.push_back(new MemoryManagerLoadGiver(
- manager, *allocType, priority, min, max, 10));
- loadGivers.back()->start(pool);
- }
- FastOS_Thread::Sleep(stressTimeMS);
- }
- FastOS_Thread::Sleep(5 * stressTimeMS);
- uint64_t okTotal = 0, failedTotal = 0, reducedTotal = 0;
- for (uint32_t i = 0; i < loadGivers.size(); i++) {
- /*
- fprintf(stderr, "%d %s-%u: Failed %d, ok %d, reduced %d\n",
- i, loadGivers[i]->_type.getName().c_str(),
- uint32_t(loadGivers[i]->_priority),
- loadGivers[i]->_failed, loadGivers[i]->_ok,
- loadGivers[i]->_reduced); // */
- okTotal += loadGivers[i]->_ok;
- failedTotal += loadGivers[i]->_failed;
- reducedTotal += loadGivers[i]->_reduced;
- }
- for (uint32_t i = 0; i < loadGivers.size(); i++) loadGivers[i]->stop();
- for (uint32_t i = 0; i < loadGivers.size(); i++) loadGivers[i]->join();
- pool.Close();
-
- /*
- bool verbose = false;
- std::cerr << "\n\nMemory allocations at end of load:\n";
- manager.print(std::cerr, verbose, ""); // */
-
- for (uint32_t i = 0; i < loadGivers.size(); i++) {
- loadGivers[i]->_tokens.clear();
- }
- for (uint32_t i = 0; i < loadGivers.size(); i++) {
- delete loadGivers[i];
- }
- loadGivers.clear();
-
- //std::cerr << "\n\nMemory allocations at end of testl:\n";
- //manager.print(std::cerr, verbose, "");
-
- std::cerr << "\n Managed " << std::fixed
- << (okTotal / (stressTimeMS / 1000))
- << " ok, " << (failedTotal / (stressTimeMS / 1000))
- << " failed and " << (reducedTotal / (stressTimeMS / 1000))
- << " reduced allocations/s.\n ";
-
- MemoryState state(clock, 1);
- manager.getState(state);
- const MemoryState::SnapShot& current(state.getCurrentSnapshot());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUserCount());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSize());
- CPPUNIT_ASSERT_EQUAL(uint64_t(0), current.getUsedSizeIgnoringCache());
-}
-
-} // defaultimplementation
-} // framework
-} // storage
diff --git a/storageframework/src/tests/memory/memorystatetest.cpp b/storageframework/src/tests/memory/memorystatetest.cpp
deleted file mode 100644
index cd565718632..00000000000
--- a/storageframework/src/tests/memory/memorystatetest.cpp
+++ /dev/null
@@ -1,172 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
-#include <vespa/storageframework/defaultimplementation/memory/memorystate.h>
-#include <vespa/vdstestlib/cppunit/macros.h>
-#include <vespa/vespalib/util/exceptions.h>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-struct MemoryStateTest : public CppUnit::TestFixture
-{
-
- void testBasics();
-
- CPPUNIT_TEST_SUITE(MemoryStateTest);
- CPPUNIT_TEST(testBasics); // Fails sometimes, test needs rewrite.
- CPPUNIT_TEST_SUITE_END();
-};
-
-CPPUNIT_TEST_SUITE_REGISTRATION(MemoryStateTest);
-
-class SimpleMemoryManager : public framework::MemoryManagerInterface
-{
-private:
- std::map<std::string, framework::MemoryAllocationType> _types;
-
-public:
- void setMaximumMemoryUsage(uint64_t max) override { (void) max; }
-
- const framework::MemoryAllocationType&
- registerAllocationType(const framework::MemoryAllocationType& type) override {
- _types[type.getName()] = type;
- return _types[type.getName()];
- }
-
- const framework::MemoryAllocationType&
- getAllocationType(const std::string& name) const override {
- std::map<std::string, framework::MemoryAllocationType>::const_iterator iter =
- _types.find(name);
-
- if (iter == _types.end()) {
- throw vespalib::IllegalArgumentException("Allocation type not found: " + name);
- }
-
- return iter->second;
- }
-
- std::vector<const MemoryAllocationType*> getAllocationTypes() const override {
- std::vector<const MemoryAllocationType*> types;
- for(std::map<std::string, framework::MemoryAllocationType>
- ::const_iterator it = _types.begin(); it != _types.end(); ++it)
- {
- types.push_back(&it->second);
- }
- return types;
- }
-
- framework::MemoryToken::UP allocate(const framework::MemoryAllocationType&,
- uint64_t,
- uint64_t,
- uint8_t,
- framework::ReduceMemoryUsageInterface*) override
- {
- return framework::MemoryToken::UP();
- }
-
- uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override {
- (void) priority;
- return 0;
- }
-};
-
-void
-MemoryStateTest::testBasics()
-{
- SimpleMemoryManager manager;
-
- const MemoryAllocationType& putAlloc(manager.registerAllocationType(
- MemoryAllocationType("MESSAGE_PUT", MemoryAllocationType::EXTERNAL_LOAD)));
- const MemoryAllocationType& getAlloc(manager.registerAllocationType(
- MemoryAllocationType("MESSAGE_GET", MemoryAllocationType::EXTERNAL_LOAD)));
- const MemoryAllocationType& blockAlloc(manager.registerAllocationType(
- MemoryAllocationType("MESSAGE_DOCBLOCK")));
- const MemoryAllocationType& databaseAlloc(manager.registerAllocationType(
- MemoryAllocationType("DATABASE")));
- const MemoryAllocationType& cacheAlloc(manager.registerAllocationType(
- MemoryAllocationType("SLOTFILE_CACHE", MemoryAllocationType::CACHE)));
-
- uint32_t maxMemory = 1024;
-
- RealClock clock;
- MemoryState state1(clock, maxMemory);
- MemoryState state2(clock, maxMemory);
-
- state1.setMinJumpToUpdateMax(50);
-
- state1.addToEntry(putAlloc, 100, 10,
- MemoryState::GOT_MAX, false);
- state1.addToEntry(putAlloc, 100, 60,
- MemoryState::GOT_MAX, false);
- state1.addToEntry(blockAlloc,
- 200, 20,
- MemoryState::GOT_MIN, false);
- state1.addToEntry(getAlloc, 0, 15,
- MemoryState::DENIED, false, 0);
- state1.addToEntry(databaseAlloc, 150, 0,
- MemoryState::DENIED, true, 1);
- state1.addToEntry(cacheAlloc, 45, 0,
- MemoryState::GOT_MAX, true, 1);
-
- state2.addToEntry(putAlloc, 50, 10,
- MemoryState::GOT_MIN, false);
- state2.addToEntry(putAlloc, 20, 40,
- MemoryState::GOT_MIN, false);
-
- state1.removeFromEntry(databaseAlloc, 25, 0, 0);
- state1.removeFromEntry(putAlloc, 100, 60);
-
- MemoryState::SnapShot state3;
- state3 = state1.getMaxSnapshot();
- state3 += state2.getMaxSnapshot();
-
- std::string expected;
- expected =
- "\n"
- "MemoryState(Max memory: 1024) {\n"
- " Current: SnapShot(Used 470, w/o cache 425) {\n"
- " Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n"
- " DATABASE(0): Used(125 B / 1) Stats(1, 0, 0, 1, 1)\n"
- " MESSAGE_DOCBLOCK(20): Used(200 B / 1) Stats(1, 0, 1, 0, 0)\n"
- " MESSAGE_GET(15): Used(0 B / 0) Stats(1, 0, 0, 1, 0)\n"
- " MESSAGE_PUT(10): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n"
- " MESSAGE_PUT(60): Used(0 B / 0) Stats(1, 1, 0, 0, 0)\n"
- " SLOTFILE_CACHE(0): Used(45 B / 1) Stats(1, 1, 0, 0, 1)\n"
- " }\n"
- " Max: SnapShot(Used 550, w/o cache 550) {\n"
- " Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n"
- " DATABASE(0): Used(150 B / 1) Stats(1, 0, 0, 1, 1)\n"
- " MESSAGE_DOCBLOCK(20): Used(200 B / 1) Stats(1, 0, 1, 0, 0)\n"
- " MESSAGE_GET(15): Used(0 B / 0) Stats(1, 0, 0, 1, 0)\n"
- " MESSAGE_PUT(10): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n"
- " MESSAGE_PUT(60): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n"
- " }\n"
- "}";
-
- CPPUNIT_ASSERT_EQUAL(expected, "\n" + state1.toString(true));
- expected = "\n"
-"MemoryState(Max memory: 1024) {\n"
-" Current: SnapShot(Used 70, w/o cache 70) {\n"
-" Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n"
-" MESSAGE_PUT(10): Used(50 B / 1) Stats(1, 0, 1, 0, 0)\n"
-" MESSAGE_PUT(40): Used(20 B / 1) Stats(1, 0, 1, 0, 0)\n"
-" }\n"
-"}";
- CPPUNIT_ASSERT_EQUAL(expected, "\n" + state2.toString(true));
- expected = "\n"
-"SnapShot(Used 550, w/o cache 550) {\n"
-" Type(Pri): Used(Size/Allocs) Stats(Allocs, Wanted, Min, Denied, Forced)\n"
-" DATABASE(0): Used(150 B / 1) Stats(1, 0, 0, 1, 1)\n"
-" MESSAGE_DOCBLOCK(20): Used(200 B / 1) Stats(1, 0, 1, 0, 0)\n"
-" MESSAGE_GET(15): Used(0 B / 0) Stats(1, 0, 0, 1, 0)\n"
-" MESSAGE_PUT(10): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n"
-" MESSAGE_PUT(60): Used(100 B / 1) Stats(1, 1, 0, 0, 0)\n"
-"}";
- CPPUNIT_ASSERT_EQUAL(expected, "\n" + state3.toString(true));
-}
-
-} // defaultimplementation
-} // framework
-} // storage
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt b/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt
index 25e7a9812cf..ea6a7e15543 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt
@@ -3,7 +3,6 @@ vespa_add_library(storageframework_defaultimplementation
SOURCES
$<TARGET_OBJECTS:storageframework_clockimpl>
$<TARGET_OBJECTS:storageframework_componentimpl>
- $<TARGET_OBJECTS:storageframework_memoryimpl>
$<TARGET_OBJECTS:storageframework_threadimpl>
INSTALL lib64
DEPENDS
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp
index a03f79b008d..84b091d25e3 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp
@@ -63,16 +63,6 @@ ComponentRegisterImpl::setMetricManager(metrics::MetricManager& mm)
}
void
-ComponentRegisterImpl::setMemoryManager(MemoryManagerInterface& mm)
-{
- vespalib::LockGuard lock(_componentLock);
- _memoryManager = &mm;
- for (uint32_t i=0; i<_components.size(); ++i) {
- _components[i]->setMemoryManager(mm);
- }
-}
-
-void
ComponentRegisterImpl::setClock(Clock& c)
{
vespalib::LockGuard lock(_componentLock);
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h
index 7319a1e7d57..3e3dd08e1df 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h
@@ -71,7 +71,6 @@ public:
void requestShutdown(vespalib::stringref reason) override;
void setMetricManager(metrics::MetricManager&);
- void setMemoryManager(MemoryManagerInterface&);
void setClock(Clock&);
void setThreadPool(ThreadPool&);
void setUpgradeFlag(UpgradeFlags flag);
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp
index 0f4d0ebe070..5efd638ec26 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp
@@ -7,13 +7,9 @@ namespace storage::framework::defaultimplementation {
TestComponentRegister::TestComponentRegister(ComponentRegisterImpl::UP compReg)
: _compReg(std::move(compReg)),
_clock(),
- _threadPool(_clock),
- _memoryManager()
+ _threadPool(_clock)
{
assert(_compReg.get() != 0);
- // Set a memory manager, so users can register memory types and
- // ask for memory.
- _compReg->setMemoryManager(_memoryManager);
// Set a fake clock, giving test control of clock
_compReg->setClock(_clock);
// Set a thread pool so components can make threads in tests.
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h
index 5226765dc40..4983d5dcdfc 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h
@@ -15,7 +15,6 @@
#include "componentregisterimpl.h"
#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h>
-#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h>
#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h>
namespace storage::framework::defaultimplementation {
@@ -24,7 +23,6 @@ class TestComponentRegister {
ComponentRegisterImpl::UP _compReg;
FakeClock _clock;
ThreadPoolImpl _threadPool;
- NoMemoryManager _memoryManager;
public:
TestComponentRegister(ComponentRegisterImpl::UP compReg);
@@ -34,7 +32,6 @@ public:
FakeClock& getClock() { return _clock; }
ThreadPoolImpl& getThreadPoolImpl() { return _threadPool; }
FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); }
- NoMemoryManager& getMemoryManager() { return _memoryManager; }
};
}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore b/storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore
deleted file mode 100644
index 7e7c0fe7fae..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/.depend
-/Makefile
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt b/storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt
deleted file mode 100644
index e0aa10a6716..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(storageframework_memoryimpl OBJECT
- SOURCES
- memorymanager.cpp
- simplememorylogic.cpp
- memorystate.cpp
- prioritymemorylogic.cpp
- nomemorymanager.cpp
- DEPENDS
-)
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h
deleted file mode 100644
index 5af2c9ff929..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/emptymemorylogic.h
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <storage/memorymanager/memorymanager.h>
-#include <storage/memorymanager/memorystate.h>
-
-namespace storage {
-
-class EmptyMemoryLogic : public AllocationLogic
-{
-private:
- MemoryState _state;
-
-public:
- EmptyMemoryLogic() : _state(100) {}
-
- virtual void getState(MemoryState& state, bool resetMax) {
- state = _state;
- if (resetMax) _state.resetMax();
- }
-
- virtual MemoryToken::UP allocate(
- const AllocationType& type,
- uint64_t, uint64_t max,
- storage::api::StorageMessage::Priority p,
- ReduceMemoryUsageInterface* = 0) {
- return MemoryToken::UP(
- new MemoryToken(*this, type, max, p));
- }
-
- virtual bool resize(MemoryToken& token, uint64_t min, uint64_t max) {
- setTokenSize(token, max);
- return true;
- }
-
- virtual void freeToken(MemoryToken&) {}
-
- virtual void print(std::ostream& out, bool verbose,
- const std::string& indent) const
- {
- (void) verbose; (void) indent;
- out << "EmptyMemoryLogic()";
- }
-};
-
-} // storage
-
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp
deleted file mode 100644
index 5af222d4b36..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.cpp
+++ /dev/null
@@ -1,157 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memorymanager.h"
-#include "memorystate.h"
-#include <vespa/vespalib/util/exceptions.h>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-MemoryTokenImpl::MemoryTokenImpl(AllocationLogic& logic,
- const MemoryAllocationType& type,
- uint64_t allocated,
- uint8_t p,
- ReduceMemoryUsageInterface* reducer)
- : _logic(logic),
- _reducer(reducer),
- _currentlyAllocated(allocated),
- _allocCount(1),
- _type(type),
- _priority(p)
-{
-}
-
-MemoryTokenImpl::~MemoryTokenImpl()
-{
- _logic.freeToken(*this);
-}
-
-bool
-MemoryTokenImpl::resize(uint64_t min, uint64_t max)
-{
- return _logic.resize(*this, min, max);
-}
-
-void
-MemoryTokenImpl::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- (void) verbose; (void) indent;
- out << "MemoryToken(" << _type.getName() << ": Allocated(" << _allocCount << " - "
- << _currentlyAllocated << ")";
-}
-
-AllocationLogic::~AllocationLogic()
-{
-}
-
-MemoryToken::UP
-AllocationLogic::allocate(const MemoryAllocationType& type,
- uint64_t min,
- uint64_t max,
- uint8_t priority,
- ReduceMemoryUsageInterface* reducer)
-{
- MemoryToken::UP token(allocate(type, priority, reducer));
- assert(token.get());
- if (!resize(*token, min, max, 1)) token.reset();
- return token;
-}
-
-bool
-AllocationLogic::resize(MemoryTokenImpl& token, uint64_t min, uint64_t max)
-{
- return resize(token, min, max, 0);
-}
-
-MemoryManager::MemoryManager(AllocationLogic::UP logic)
- : _logic(std::move(logic))
-{
- if (_logic.get() == 0) {
- throw vespalib::IllegalArgumentException(
- "Needs a real logic class to run. (Got null pointer)",
- VESPA_STRLOC);
- }
-}
-
-MemoryManager::~MemoryManager()
-{
-}
-
-void
-MemoryManager::setMaximumMemoryUsage(uint64_t max)
-{
- _logic->setMaximumMemoryUsage(max);
-}
-
-void
-MemoryManager::getState(MemoryState& state, bool resetMax)
-{
- return _logic->getState(state, resetMax);
-}
-
-const MemoryAllocationType&
-MemoryManager::registerAllocationType(const MemoryAllocationType& type)
-{
- vespalib::LockGuard lock(_typeLock);
- _types[type.getName()] = MemoryAllocationType::UP(
- new MemoryAllocationType(type));
- return *_types[type.getName()];
-}
-
-const MemoryAllocationType&
-MemoryManager::getAllocationType(const std::string& name) const
-{
- vespalib::LockGuard lock(_typeLock);
- std::map<std::string, MemoryAllocationType::UP>::const_iterator it(
- _types.find(name));
- if (it == _types.end()) {
- throw vespalib::IllegalArgumentException(
- "Allocation type not found: " + name, VESPA_STRLOC);
- }
- return *it->second;
-}
-
-std::vector<const MemoryAllocationType*>
-MemoryManager::getAllocationTypes() const
-{
- vespalib::LockGuard lock(_typeLock);
- std::vector<const MemoryAllocationType*> types;
- for(std::map<std::string, MemoryAllocationType::UP>::const_iterator it
- = _types.begin(); it != _types.end(); ++it)
- {
- types.push_back(it->second.get());
- }
- return types;
-}
-
-MemoryToken::UP
-MemoryManager::allocate(const MemoryAllocationType& type,
- uint64_t min,
- uint64_t max,
- uint8_t p,
- ReduceMemoryUsageInterface* i)
-{
- return _logic->allocate(type, min, max, p, i);
-}
-
-uint64_t
-MemoryManager::getMemorySizeFreeForPriority(uint8_t priority) const
-{
- return _logic->getMemorySizeFreeForPriority(priority);
-}
-
-void
-MemoryManager::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- (void) verbose; (void) indent;
- out << "Memory Manager {" << "\n" << indent << " ";
- _logic->print(out, verbose, indent + " ");
- out << "\n" << indent << "}";
-}
-
-} // defaultimplementation
-} // framework
-} // storage
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h
deleted file mode 100644
index 86a2920095e..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h
+++ /dev/null
@@ -1,156 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::MemoryManager
- *
- * \brief Utility for tracking memory usage on distributor and storage nodes.
- *
- * The memory manager is responsible for limiting the memory allocated by
- * various users within VDS storage and distributor nodes, such that these
- * nodes don't use more memory than they can use, to avoid swapping.
- *
- * It will produce a status page to view big memory users, and give some
- * historic data. It will track memory users and give them less and less memory
- * as closer we are to utilizing all memory we are able to use. When getting
- * close to full it will deny memory allocations to incoming commands that wants
- * to use additional memory in able to complete the operation.
- *
- * The main class here defines the interface the client has to worry about. It
- * should thus not point to the implementation in any way.
- *
- */
-
-#pragma once
-
-#include <map>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
-#include <vespa/vespalib/util/printable.h>
-#include <vespa/vespalib/util/sync.h>
-
-namespace storage::framework::defaultimplementation {
-
-class MemoryManager;
-class AllocationLogic;
-class MemoryState;
-
-class MemoryTokenImpl : public vespalib::Printable,
- public MemoryToken
-{
- friend class AllocationLogic;
-
- AllocationLogic& _logic;
- ReduceMemoryUsageInterface* _reducer;
- uint64_t _currentlyAllocated;
- uint32_t _allocCount;
- const MemoryAllocationType& _type;
- uint8_t _priority;
-
-public:
- typedef std::unique_ptr<MemoryTokenImpl> UP;
-
- MemoryTokenImpl(const MemoryTokenImpl &) = delete;
- MemoryTokenImpl & operator = (const MemoryTokenImpl &) = delete;
- MemoryTokenImpl(AllocationLogic& logic,
- const MemoryAllocationType& type,
- uint64_t allocated,
- uint8_t priority,
- ReduceMemoryUsageInterface* = 0);
-
- ~MemoryTokenImpl();
-
- uint64_t getSize() const override { return _currentlyAllocated; }
- uint64_t getAllocationCount() const { return _allocCount; }
- const MemoryAllocationType& getType() const { return _type; }
- ReduceMemoryUsageInterface* getReducer() const { return _reducer; }
-
- uint8_t getPriority() const { return _priority; }
-
- bool resize(uint64_t min, uint64_t max) override;
-
- void print(std::ostream& out, bool verbose,
- const std::string& indent) const override;
-};
-
-class AllocationLogic : public vespalib::Printable
-{
-protected:
- /**
- * MemoryTokens are friends with this class, such that logic classes
- * can use this function to alter token size, without function for that
- * being public.
- */
- void setTokenSize(MemoryTokenImpl& token, uint64_t size)
- { token._currentlyAllocated = size; }
-
- virtual MemoryToken::UP allocate(const MemoryAllocationType&,
- uint8_t priority,
- ReduceMemoryUsageInterface*) = 0;
- virtual bool resize(MemoryToken& token, uint64_t min, uint64_t max,
- uint32_t allocationCounts) = 0;
-public:
- typedef std::unique_ptr<AllocationLogic> UP;
- virtual ~AllocationLogic() = 0;
-
- virtual void setMaximumMemoryUsage(uint64_t max) = 0;
-
- virtual void getState(MemoryState&, bool resetMax = false) = 0;
-
- /**
- * Decide how much to allocate for this request. Should be between min
- * and max, unless it's of a type that can be denied (such as external
- * requests), in which case we can also deny allocation by returning a null
- * token.
- */
- MemoryToken::UP allocate(const MemoryAllocationType&,
- uint64_t min,
- uint64_t max,
- uint8_t priority,
- ReduceMemoryUsageInterface* = 0);
- /**
- * Resize the size in a token. If more memory is requested, then it might
- * fail. The sizes given in min and max is given as total min and max,
- * including any memory you may already have. If successful, the logic will
- * have added this size to the token passed in.
- */
- bool resize(MemoryTokenImpl& token, uint64_t min, uint64_t max);
-
- // Called by token destructor to free up tracked resources
- virtual void freeToken(MemoryTokenImpl& token) = 0;
-
- virtual uint64_t getMemorySizeFreeForPriority(uint8_t priority) const = 0;
-
- // vespalib::Printable implementation
- virtual void print(std::ostream& out, bool verbose,
- const std::string& indent) const override = 0;
-};
-
-class MemoryManager : public vespalib::Printable,
- public MemoryManagerInterface
-{
- AllocationLogic::UP _logic;
- vespalib::Lock _typeLock;
- std::map<std::string, MemoryAllocationType::UP> _types;
-
-public:
- typedef std::unique_ptr<MemoryManager> UP;
-
- MemoryManager(AllocationLogic::UP);
- ~MemoryManager();
-
- void setMaximumMemoryUsage(uint64_t max) override;
- virtual void getState(MemoryState& state, bool resetMax = false);
-
- const MemoryAllocationType&registerAllocationType(const MemoryAllocationType& type) override;
- const MemoryAllocationType&getAllocationType(const std::string& name) const override;
-
- std::vector<const MemoryAllocationType*> getAllocationTypes() const override;
-
- MemoryToken::UP allocate(const MemoryAllocationType&, uint64_t min, uint64_t max,
- uint8_t p, ReduceMemoryUsageInterface* = 0) override;
-
- uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override;
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp
deleted file mode 100644
index 2351edd267c..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp
+++ /dev/null
@@ -1,225 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memorystate.h"
-#include <sstream>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".memory.state");
-
-namespace storage::framework::defaultimplementation {
-
-MemoryState::Entry::Entry()
- : _currentUsedSize(0),
- _totalUserCount(0),
- _currentUserCount(0),
- _wantedCount(0),
- _minimumCount(0),
- _deniedCount(0),
- _forcedBeyondMaximumCount(0)
-{
-}
-
-void
-MemoryState::Entry::operator+=(const Entry& other)
-{
- _currentUsedSize += other._currentUsedSize;
- _currentUserCount += other._currentUserCount;
- _totalUserCount += other._totalUserCount;
- _wantedCount += other._wantedCount;
- _minimumCount += other._minimumCount;
- _deniedCount += other._deniedCount;
- _forcedBeyondMaximumCount += other._forcedBeyondMaximumCount;
-}
-
-MemoryState::SnapShot&
-MemoryState::SnapShot::operator+=(const MemoryState::SnapShot& other)
-{
- for (AllocationMap::const_iterator it = other._allocations.begin();
- it != other._allocations.end(); ++it)
- {
- PriorityMap& map(_allocations[it->first]);
- for (PriorityMap::const_iterator it2 = it->second.begin();
- it2 != it->second.end(); ++it2)
- {
- Entry& entry(map[it2->first]);
- entry += it2->second;
- }
- }
- return *this;
-}
-
-uint64_t
-MemoryState::SnapShot::getUserCount() const
-{
- uint64_t count = 0;
- for (AllocationMap::const_iterator it = _allocations.begin();
- it != _allocations.end(); ++it)
- {
- for (PriorityMap::const_iterator it2 = it->second.begin();
- it2 != it->second.end(); ++it2)
- {
- count += it2->second._currentUserCount;
- }
- }
- return count;
-}
-
-MemoryState::MemoryState(Clock& clock, uint64_t maxMemory)
- : _clock(&clock),
- _maxMemory(maxMemory),
- _current(),
- _max(),
- _minJumpToUpdateMax(10 * 1024 * 1024)
-{
-}
-
-MemoryState::MemoryState(const MemoryState &) = default;
-MemoryState & MemoryState::operator = (const MemoryState &) = default;
-
-MemoryState::~MemoryState() {}
-
-void
-MemoryState::addToEntry(const MemoryAllocationType& type, uint64_t memory,
- uint8_t priority,
- AllocationResult result, bool forcedAllocation,
- uint64_t allocationCounts)
-{
- LOG(spam, "Allocating memory %s - %lu bytes at priority %u. "
- "Count %lu.",
- type.getName().c_str(), memory, priority, allocationCounts);
- PriorityMap& map(_current._allocations[&type]);
- Entry& e(map[priority]);
- e._currentUsedSize += memory;
- e._totalUserCount += allocationCounts;
- if (allocationCounts == 0) {
- // Resizes adding no more users still count as another total
- // allocation attempt.
- ++e._totalUserCount;
- }
- e._currentUserCount += allocationCounts;
- switch (result) {
- case GOT_MAX: ++e._wantedCount; break;
- case GOT_MIN: ++e._minimumCount; break;
- case DENIED: ++e._deniedCount; break;
- }
- if (forcedAllocation) ++e._forcedBeyondMaximumCount;
- _current._usedMemory += memory;
- if (!type.isCache()) {
- _current._usedWithoutCache += memory;
- }
- if (_current._usedWithoutCache
- > _max._usedWithoutCache + _minJumpToUpdateMax)
- {
- LOG(spam, "Updating max to current %lu bytes of memory used",
- _current._usedWithoutCache);
- _max = _current;
- _max._timeTaken = _clock->getTimeInSeconds();
- }
-}
-
-void
-MemoryState::removeFromEntry(const MemoryAllocationType& type, uint64_t memory,
- uint8_t priority,
- uint64_t allocationCounts)
-{
- LOG(spam, "Freeing memory %s - %lu bytes at priority %u. "
- "Count %lu.",
- type.getName().c_str(), memory, priority, allocationCounts);
- PriorityMap& map(_current._allocations[&type]);
- Entry& e(map[priority]);
- e._currentUsedSize -= memory;
- e._currentUserCount -= allocationCounts;
- _current._usedMemory -= memory;
- if (!type.isCache()) {
- _current._usedWithoutCache -= memory;
- }
-}
-
-void
-MemoryState::Entry::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- (void) verbose; (void) indent;
- std::ostringstream ost;
- ost << "Used(" << _currentUsedSize << " B / "
- << _currentUserCount << ") ";
- for (uint32_t i=ost.str().size(); i<20; ++i) {
- ost << " ";
- }
-
- out << ost.str()
- << "Stats(" << _totalUserCount
- << ", " << _wantedCount
- << ", " << _minimumCount
- << ", " << _deniedCount
- << ", " << _forcedBeyondMaximumCount << ")";
-}
-
-namespace {
- void printAllocations(std::ostream& out,
- const MemoryState::AllocationMap& map,
- const std::string& indent)
- {
- std::map<std::string, std::string> allocs;
-
- for (MemoryState::AllocationMap::const_iterator it = map.begin();
- it != map.end(); ++it)
- {
- for (MemoryState::PriorityMap::const_iterator it2
- = it->second.begin(); it2 != it->second.end(); ++it2)
- {
- std::ostringstream name;
- name << it->first->getName() << "("
- << static_cast<uint16_t>(it2->first) << "): ";
- for (uint32_t i=name.str().size(); i<25; ++i) {
- name << " ";
- }
-
- std::ostringstream tmp;
- it2->second.print(tmp, true, indent + " ");
-
- allocs[name.str()] = tmp.str();
- }
- }
-
- for (std::map<std::string, std::string>::const_iterator it
- = allocs.begin(); it != allocs.end(); ++it)
- {
- out << "\n" << indent << " " << it->first << it->second;
- }
- }
-}
-
-void
-MemoryState::SnapShot::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- out << "SnapShot(Used " << _usedMemory << ", w/o cache "
- << _usedWithoutCache;
- if (verbose) {
- out << ") {";
- if (_usedMemory > 0) {
- out << "\n" << indent << " Type(Pri): Used(Size/Allocs) "
- << "Stats(Allocs, Wanted, Min, Denied, Forced)";
- }
- printAllocations(out, _allocations, indent);
- out << "\n" << indent << "}";
- }
-}
-
-void
-MemoryState::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- bool maxSet = (_max._usedWithoutCache > _current._usedWithoutCache);
- out << "MemoryState(Max memory: " << _maxMemory << ") {"
- << "\n" << indent << " Current: ";
- _current.print(out, verbose, indent + " ");
- if (maxSet) {
- out << "\n" << indent << " Max: ";
- _max.print(out, verbose, indent + " ");
- }
- out << "\n" << indent << "}";
-}
-
-}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h
deleted file mode 100644
index 3d670d43844..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h
+++ /dev/null
@@ -1,139 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::MemoryState
- *
- * \brief Shows the state of current memory users
- *
- */
-
-#pragma once
-
-#include "memorymanager.h"
-#include <vespa/storageframework/generic/clock/clock.h>
-#include <vespa/vespalib/util/sync.h>
-
-namespace storage::framework::defaultimplementation {
-
-class MemoryState : public vespalib::Printable {
-public:
- struct Entry {
- // Total number of bytes allocated to this entry right now
- uint64_t _currentUsedSize;
- // Total number of allocations done on this entry
- uint64_t _totalUserCount;
- // Total number of allocations for this entry right now
- uint32_t _currentUserCount;
- // Amount of times this entry has gotten all the memory it wanted
- uint32_t _wantedCount;
- // Amount of times this entry has gotten less than all the memory
- // it wanted
- uint32_t _minimumCount;
- // Amount of times this entry has been denied getting memory
- uint32_t _deniedCount;
- // Amount of times this entry has forced memory allocations beyond
- // the maximum
- uint32_t _forcedBeyondMaximumCount;
-
- Entry();
-
- void print(std::ostream& out, bool verbose,
- const std::string& indent) const;
-
- /**
- * Set this instances counts to the counts from the other entry.
- */
- void transferCounts(const Entry& other);
-
- void operator+=(const Entry& other);
- };
-
- typedef std::map<uint8_t, Entry> PriorityMap;
- typedef std::map<const MemoryAllocationType*, PriorityMap> AllocationMap;
-
- /**
- * A snapshot contains data for either current or max seen data.
- * When a new maximum is seen, current is copied to max.
- */
- class SnapShot : public vespalib::Printable {
- friend class MemoryState;
-
- uint64_t _usedMemory;
- uint64_t _usedWithoutCache;
- SecondTime _timeTaken;
- AllocationMap _allocations;
-
- public:
- SnapShot() : vespalib::Printable() { clear(); }
- SnapShot(const SnapShot& o) : vespalib::Printable() { (*this) = o; }
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- void clear() {
- _usedMemory = 0;
- _usedWithoutCache = 0;
- _timeTaken.setTime(0);
- _allocations.clear();
- }
-
- SnapShot& operator=(const SnapShot& other) {
- _usedMemory = other._usedMemory;
- _usedWithoutCache = other._usedWithoutCache;
- _timeTaken = other._timeTaken;
- _allocations = other._allocations;
- return *this;
- }
-
- SnapShot& operator+=(const SnapShot& other);
-
- const AllocationMap& getAllocations() const { return _allocations; }
- uint64_t getUsedSize() const { return _usedMemory; }
- uint64_t getUsedSizeIgnoringCache() const { return _usedWithoutCache; }
- uint64_t getUserCount() const;
- };
-
-private:
- Clock* _clock;
- uint64_t _maxMemory;
- SnapShot _current;
- SnapShot _max;
- uint32_t _minJumpToUpdateMax;
-
-public:
- MemoryState(Clock& clock, uint64_t maxMemory);
- MemoryState(const MemoryState &);
- MemoryState & operator = (const MemoryState &);
- MemoryState(MemoryState &&) = default;
- MemoryState & operator = (MemoryState &&) = default;
- ~MemoryState();
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- void setMaximumMemoryUsage(uint64_t max) { _maxMemory = max; }
- void setMinJumpToUpdateMax(uint32_t bytes) { _minJumpToUpdateMax = bytes; }
-
- enum AllocationResult { GOT_MAX, GOT_MIN, DENIED };
- void addToEntry(const MemoryAllocationType& type, uint64_t memory,
- uint8_t priority,
- AllocationResult result, bool forcedAllocation = false,
- uint64_t allocationCounts = 1);
-
- void removeFromEntry(const MemoryAllocationType& type, uint64_t memory,
- uint8_t priority,
- uint64_t allocationCounts = 1);
- void resetMax() {
- _max = _current;
- _max._timeTaken = _clock->getTimeInSeconds();
- }
-
- const SnapShot& getCurrentSnapshot() const { return _current; }
- const SnapShot& getMaxSnapshot() const { return _max; }
-
-
- uint64_t getTotalSize() const { return _maxMemory; }
- uint64_t getFreeSize() const {
- return _maxMemory > _current._usedMemory
- ? _maxMemory - _current._usedMemory : 0;
- }
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp
deleted file mode 100644
index 16e03240a4f..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "nomemorymanager.h"
-#include <vespa/vespalib/util/exceptions.h>
-
-namespace storage::framework::defaultimplementation {
-
-NoMemoryManager::~NoMemoryManager() {}
-
-const MemoryAllocationType&
-NoMemoryManager::registerAllocationType(const MemoryAllocationType& type)
-{
- vespalib::LockGuard lock(_typeLock);
- _types[type.getName()] = MemoryAllocationType::UP(
- new MemoryAllocationType(type));
- return *_types[type.getName()];
-}
-
-const MemoryAllocationType&
-NoMemoryManager::getAllocationType(const std::string& name) const
-{
- vespalib::LockGuard lock(_typeLock);
- std::map<std::string, MemoryAllocationType::UP>::const_iterator it(
- _types.find(name));
- if (it == _types.end()) {
- throw vespalib::IllegalArgumentException(
- "Allocation type not found: " + name, VESPA_STRLOC);
- }
- return *it->second;
-}
-
-std::vector<const MemoryAllocationType*>
-NoMemoryManager::getAllocationTypes() const
-{
- vespalib::LockGuard lock(_typeLock);
- std::vector<const MemoryAllocationType*> types;
- for(std::map<std::string, MemoryAllocationType::UP>::const_iterator it
- = _types.begin(); it != _types.end(); ++it)
- {
- types.push_back(it->second.get());
- }
- return types;
-}
-
-}
-
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h
deleted file mode 100644
index 7f90b1360b8..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/nomemorymanager.h
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::NoMemoryManager
- *
- * \brief Memory manager that gives out max memory to everyone.
- *
- * Memory manager to use for testing and for apps not wanting to track memory.
- * This manager will merely give out max to everyone who asks and not even keep
- * track of anything.
- */
-
-#pragma once
-
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
-#include <vespa/vespalib/util/printable.h>
-#include <vespa/vespalib/util/sync.h>
-#include <map>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-class SimpleMemoryTokenImpl : public MemoryToken
-{
- uint64_t _allocated;
-
-public:
- SimpleMemoryTokenImpl(const SimpleMemoryTokenImpl &) = delete;
- SimpleMemoryTokenImpl & operator = (const SimpleMemoryTokenImpl &) = delete;
- SimpleMemoryTokenImpl(uint64_t allocated) : _allocated(allocated) {}
-
- uint64_t getSize() const override { return _allocated; }
- bool resize(uint64_t /* min */, uint64_t max) override { _allocated = max; return true; }
-};
-
-class NoMemoryManager : public MemoryManagerInterface
-{
- vespalib::Lock _typeLock;
- std::map<std::string, MemoryAllocationType::UP> _types;
-
-public:
- typedef std::unique_ptr<NoMemoryManager> UP;
-
- ~NoMemoryManager();
-
- void setMaximumMemoryUsage(uint64_t) override {}
- const MemoryAllocationType & registerAllocationType(const MemoryAllocationType& type) override;
- const MemoryAllocationType & getAllocationType(const std::string& name) const override;
-
- MemoryToken::UP allocate(const MemoryAllocationType&, uint64_t /* min */, uint64_t max,
- uint8_t /* priority */, ReduceMemoryUsageInterface* = 0) override
- {
- return SimpleMemoryTokenImpl::UP(new SimpleMemoryTokenImpl(max));
- }
- uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override {
- (void) priority;
- return std::numeric_limits<uint64_t>().max();
- }
-
- std::vector<const MemoryAllocationType*> getAllocationTypes() const override;
-};
-
-} // defaultimplementation
-} // framework
-} // storage
-
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp
deleted file mode 100644
index 51bb7ba5019..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "prioritymemorylogic.h"
-
-#include <vespa/log/log.h>
-LOG_SETUP(".memory.logic.priority");
-
-namespace storage::framework::defaultimplementation {
-
-PriorityMemoryLogic::PriorityMemoryLogic(Clock& c, uint64_t maxMem)
- : SimpleMemoryLogic(c, maxMem)
-{
- LOG(debug, "Setup priority memory logic with max memory of %lu bytes", maxMem);
-}
-
-float
-PriorityMemoryLogic::getNonCacheThreshold(uint8_t priority) const
-{
- return 0.6 + ((255 - priority) / 255.0) * 0.4;
-}
-
-void
-PriorityMemoryLogic::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- out << "PriorityMemoryLogic() : ";
- SimpleMemoryLogic::print(out, verbose, indent);
-}
-
-}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h
deleted file mode 100644
index 148c4f40aee..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-/**
- * \class storage::PriorityMemoryLogic
- *
- * \brief Priority logic deciding who should get memory and how much.
- *
- */
-
-#pragma once
-
-#include "simplememorylogic.h"
-#include <vespa/vespalib/util/sync.h>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-struct PriorityMemoryLogic : public SimpleMemoryLogic
-{
- PriorityMemoryLogic(Clock&, uint64_t maxMemory);
- float getNonCacheThreshold(uint8_t priority) const override;
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-};
-
-} // defaultimplementation
-} // framework
-} // storage
-
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp
deleted file mode 100644
index 76d6f990a00..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp
+++ /dev/null
@@ -1,232 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "simplememorylogic.h"
-#include <vespa/vespalib/util/exceptions.h>
-
-#include <vespa/log/bufferedlogger.h>
-LOG_SETUP(".memory.logic.simple");
-
-namespace storage::framework::defaultimplementation {
-
-SimpleMemoryLogic::SimpleMemoryLogic(Clock& c, uint64_t maxMemory)
- : _cacheThreshold(0.98),
- _nonCacheThreshold(0.8),
- _state(c, maxMemory),
- _reducers()
-{
- LOG(debug, "Setup simple memory logic with max memory of %" PRIu64 " bytes", maxMemory);
-}
-
-SimpleMemoryLogic::~SimpleMemoryLogic()
-{
-}
-
-void
-SimpleMemoryLogic::setMaximumMemoryUsage(uint64_t max)
-{
- vespalib::LockGuard lock(_stateLock);
- _state.setMaximumMemoryUsage(max);
-}
-
-void
-SimpleMemoryLogic::getState(MemoryState& state, bool resetMax) {
- vespalib::LockGuard lock(_stateLock);
- state = _state;
- if (resetMax) _state.resetMax();
-}
-
-MemoryToken::UP
-SimpleMemoryLogic::allocate(const MemoryAllocationType& type,
- uint8_t priority,
- ReduceMemoryUsageInterface* reducer)
-{
- MemoryTokenImpl::UP token(
- new MemoryTokenImpl(*this, type, 0, priority, reducer));
- if (reducer != 0) {
- vespalib::LockGuard lock(_stateLock);
- _reducers.push_back(Reducer(*token, *reducer));
- }
- return std::move(token);
-}
-
-bool
-SimpleMemoryLogic::resize(MemoryToken& tok, uint64_t min, uint64_t max,
- uint32_t allocationCounts)
-{
- vespalib::LockGuard lock(_stateLock);
- MemoryTokenImpl& token(static_cast<MemoryTokenImpl&>(tok));
- LOG(spam, "Attempting to resize %s to size in the range %" PRIu64 " to "
- "%" PRIu64 ".", token.toString().c_str(), min, max);
- if (token.getSize() > max) { // Always safe to reduce size
- handleReduction(token, max, allocationCounts);
- return true;
- }
- // If not reducing size, calculate relative min/max values.
- uint64_t relMin = (min > token.getSize() ? min - token.getSize() : 0);
- uint64_t relMax = max - token.getSize();
- return resizeRelative(token, relMin, relMax, allocationCounts);
-}
-
-void
-SimpleMemoryLogic::handleReduction(MemoryTokenImpl& token, uint64_t max,
- uint32_t allocationCounts)
-{
- LOG(spam, "Reduzing size of token by %" PRIu64 ".",
- token.getSize() - max);
- _state.removeFromEntry(token.getType(), token.getSize() - max,
- token.getPriority(), allocationCounts);
- setTokenSize(token, max);
-}
-
-bool
-SimpleMemoryLogic::handleCacheMemoryRequest(
- MemoryTokenImpl& token, uint64_t min, uint64_t max,
- uint32_t allocationCounts)
-{
- uint64_t usedSize(_state.getCurrentSnapshot().getUsedSize());
- uint64_t thresholdSize = uint64_t(getCacheThreshold()
- * _state.getTotalSize());
- uint64_t toAllocate(thresholdSize > usedSize
- ? std::min(thresholdSize - usedSize, max)
- : 0);
- bool forced = false;
- if (token.getType().isAllocationsForced() && toAllocate < min) {
- toAllocate = min;
- forced = true;
- }
- if (toAllocate < min) {
- LOG(spam, "We cannot give more memory to cache without going above "
- "cache threshold (%" PRIu64 " B)", thresholdSize);
- _state.addToEntry(token.getType(), 0, token.getPriority(),
- MemoryState::DENIED, false, allocationCounts);
- return false;
- }
- LOG(spam, "Giving %" PRIu64 " bytes of memory to cache. (Cache threshold "
- "is %" PRIu64 ", used size is %" PRIu64 ", %" PRIu64 " bytes were "
- "always allocated to the token and it wanted memory between %"
- PRIu64 " and %" PRIu64 ".",
- toAllocate, thresholdSize, usedSize, token.getSize(), min, max);
- _state.addToEntry(token.getType(), toAllocate, token.getPriority(),
- static_cast<uint64_t>(toAllocate) >= max
- ? MemoryState::GOT_MAX : MemoryState::GOT_MIN,
- forced, allocationCounts);
- setTokenSize(token, token.getSize() + toAllocate);
- return true;
-}
-
-uint64_t
-SimpleMemoryLogic::getMemorySizeFreeForPriority(uint8_t priority) const
-{
- uint64_t usedSize(_state.getCurrentSnapshot().getUsedSizeIgnoringCache());
- uint64_t thresholdSize = uint64_t(getNonCacheThreshold(priority)
- * _state.getTotalSize());
- return (usedSize >= thresholdSize ? 0 : thresholdSize - usedSize);
-}
-
-bool
-SimpleMemoryLogic::resizeRelative(
- MemoryTokenImpl& token, uint64_t min, uint64_t max,
- uint32_t allocationCounts)
-{
- LOG(spam, "Relative resize change. Need another %zu-%zu byte of memory.",
- min, max);
- // If requester is cache, use cache threshold
- if (token.getType().isCache()) {
- return handleCacheMemoryRequest(token, min, max, allocationCounts);
- }
- // If we get here, requester is not cache.
- uint64_t usedSize(_state.getCurrentSnapshot().getUsedSizeIgnoringCache());
- uint64_t thresholdSize = uint64_t(getNonCacheThreshold(token.getPriority())
- * _state.getTotalSize());
- uint64_t toAllocate = 0;
- if (thresholdSize > usedSize) {
- toAllocate = std::min(max, thresholdSize - usedSize);
- }
- if (toAllocate < min) toAllocate = min;
- bool forced = false;
- if (usedSize + toAllocate > _state.getTotalSize()) {
- if (token.getType().isAllocationsForced()) {
- forced = true;
- } else {
- LOG(spam, "We cannot give more memory without going beyond max");
- _state.addToEntry(token.getType(), 0, token.getPriority(),
- MemoryState::DENIED, false, allocationCounts);
- return false;
- }
- }
- // External load should not fill up too much
- if (usedSize + toAllocate > thresholdSize
- && token.getType().isExternalLoad()
- && !token.getType().isAllocationsForced())
- {
- LOG(spam, "Not giving external load memory beyond threshold.");
- _state.addToEntry(token.getType(), 0, token.getPriority(),
- MemoryState::DENIED, false, allocationCounts);
- return false;
- }
- // If this puts us above max with cache, remove some cache.
- if (_state.getCurrentSnapshot().getUsedSize() + toAllocate
- > _state.getTotalSize())
- {
- uint64_t needed(_state.getCurrentSnapshot().getUsedSize()
- + toAllocate - _state.getTotalSize());
- for (uint32_t i=0; i<_reducers.size(); ++i) {
- MemoryTokenImpl& rtoken(*_reducers[i]._token);
- uint64_t reduceBy(std::min(needed, rtoken.getSize()));
- uint64_t reduced(_reducers[i]._reducer->reduceMemoryConsumption(
- rtoken, reduceBy));
- _state.removeFromEntry(rtoken.getType(), reduced,
- rtoken.getPriority(), 0);
- setTokenSize(rtoken, rtoken.getSize() - reduced);
- needed -= reduceBy;
- if (needed == 0) break;
- if (reduced < reduceBy) {
- LOG(debug, "Reducer refused to free the full %" PRIu64 " bytes "
- "requested. %" PRIu64 " bytes reduced in token %s.",
- reduceBy, reduced, rtoken.toString().c_str());
- }
- }
- }
- if (_state.getCurrentSnapshot().getUsedSize() + toAllocate
- > _state.getTotalSize())
- {
- LOGBP(debug, "Failed to free enough memory from cache. This puts us "
- "above max memory.");
- }
- LOG(spam, "Giving %" PRIu64 " bytes of memory", toAllocate);
- _state.addToEntry(token.getType(), toAllocate, token.getPriority(),
- static_cast<uint64_t>(toAllocate) >= max
- ? MemoryState::GOT_MAX : MemoryState::GOT_MIN,
- forced, allocationCounts);
- setTokenSize(token, token.getSize() + toAllocate);
- return true;
-}
-
-void
-SimpleMemoryLogic::freeToken(MemoryTokenImpl& token)
-{
- vespalib::LockGuard lock(_stateLock);
- _state.removeFromEntry(token.getType(), token.getSize(),
- token.getPriority(), token.getAllocationCount());
- if (token.getReducer() != 0) {
- std::vector<Reducer> reducers;
- reducers.reserve(_reducers.size() - 1);
- for (uint32_t i=0; i<_reducers.size(); ++i) {
- if (_reducers[i]._token != &token) reducers.push_back(_reducers[i]);
- }
- assert(reducers.size() + 1 == _reducers.size());
- reducers.swap(_reducers);
- }
-}
-
-void
-SimpleMemoryLogic::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- out << "SimpleMemoryLogic() {\n"
- << indent << " ";
- vespalib::LockGuard lock(_stateLock);
- _state.print(out, verbose, indent + " ");
-}
-
-}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h
deleted file mode 100644
index c394db40b29..00000000000
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.h
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-/**
- * \class storage::SimpleMemoryLogic
- *
- * \brief Simple logic deciding who should get memory and how much.
- *
- * There is a cache threshold. By default 98%. Cache will always get memory up
- * till this fillrate.
- *
- * There is a non-cache threshold. Non-cache memory requesters will get maximum
- * memory until threshold is reached. If getting maximum memory would go beyond
- * the non-cache threshold, the requester will get enough memory to hit the
- * threshold (if more than minimum), or get the minimum memory asked for, if
- * that doesn't put usage above 100%.
- *
- * Usage above 100% is attempted avoided by freeing cache memory. If failing to
- * free enough memory, request will fail, or minimum will be get if allocation
- * is forced such that it cannot fail. In such a case, usage may go beyond 100%.
- */
-
-#pragma once
-
-#include "memorymanager.h"
-#include "memorystate.h"
-#include <vespa/vespalib/util/sync.h>
-
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
-
-class SimpleMemoryLogic : public AllocationLogic
-{
- float _cacheThreshold;
- float _nonCacheThreshold;
- vespalib::Lock _stateLock;
- MemoryState _state;
- struct Reducer {
- MemoryTokenImpl* _token;
- ReduceMemoryUsageInterface* _reducer;
-
- Reducer() : _token(0), _reducer(0) {}
- Reducer(MemoryTokenImpl& t,
- ReduceMemoryUsageInterface& r)
- : _token(&t), _reducer(&r) {}
- };
- std::vector<Reducer> _reducers;
-
-protected:
- float getCacheThreshold() { return _cacheThreshold; }
-
- // Priority memory logic can override this to set a threshold based on
- // priority
- virtual float getNonCacheThreshold(uint8_t priority) const
- { (void) priority; return _nonCacheThreshold; }
-
-public:
- typedef std::unique_ptr<SimpleMemoryLogic> UP;
-
- SimpleMemoryLogic(Clock&, uint64_t maxMemory);
-
- ~SimpleMemoryLogic();
-
- SimpleMemoryLogic& setMinJumpToUpdateMax(uint32_t bytes) {
- _state.setMinJumpToUpdateMax(bytes);
- return *this;
- }
-
- void setMaximumMemoryUsage(uint64_t max) override;
-
- void setCacheThreshold(float limit) { _cacheThreshold = limit; }
- void setNonCacheThreshold(float limit) { _nonCacheThreshold = limit; }
-
- MemoryState& getState() { return _state; } // Not threadsafe. Unit testing.
- void getState(MemoryState& state, bool resetMax) override;
-
- MemoryToken::UP allocate(const MemoryAllocationType&, uint8_t priority,
- ReduceMemoryUsageInterface* = 0) override;
- bool resize(MemoryToken& token, uint64_t min, uint64_t max, uint32_t allocationCounts) override;
-
- void freeToken(MemoryTokenImpl& token) override;
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- virtual uint64_t getMemorySizeFreeForPriority(uint8_t priority) const override;
-
-private:
- void handleReduction(MemoryTokenImpl&, uint64_t size,
- uint32_t allocationCounts);
- bool resizeRelative(MemoryTokenImpl&, uint64_t min, uint64_t max,
- uint32_t allocationCounts);
- bool handleCacheMemoryRequest(MemoryTokenImpl&, uint64_t min, uint64_t max,
- uint32_t allocationCounts);
-};
-
-} // defaultimplementation
-} // framework
-} // storage
-
diff --git a/storageframework/src/vespa/storageframework/generic/CMakeLists.txt b/storageframework/src/vespa/storageframework/generic/CMakeLists.txt
index cf225f8d5aa..5df9963304d 100644
--- a/storageframework/src/vespa/storageframework/generic/CMakeLists.txt
+++ b/storageframework/src/vespa/storageframework/generic/CMakeLists.txt
@@ -4,7 +4,6 @@ vespa_add_library(storageframework_generic
$<TARGET_OBJECTS:storageframework_component>
$<TARGET_OBJECTS:storageframework_status>
$<TARGET_OBJECTS:storageframework_thread>
- $<TARGET_OBJECTS:storageframework_memory>
$<TARGET_OBJECTS:storageframework_clock>
INSTALL lib64
DEPENDS
diff --git a/storageframework/src/vespa/storageframework/generic/memory/.gitignore b/storageframework/src/vespa/storageframework/generic/memory/.gitignore
deleted file mode 100644
index 7e7c0fe7fae..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/.depend
-/Makefile
diff --git a/storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt b/storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt
deleted file mode 100644
index 77aa14e4ca2..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(storageframework_memory OBJECT
- SOURCES
- memorytoken.cpp
- DEPENDS
-)
diff --git a/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h b/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h
deleted file mode 100644
index e91b7946454..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::framework::MemoryAllocationType
- * \ingroup memory
- *
- * \brief Allocation types used to differ between memory manager clients.
- *
- * The different memory manager clients have different properties. It is
- * important for the memory manager to distinguish between different users in
- * order to know how to prioritize memory, and also in order to create good
- * reports on memory usage.
- *
- * An allocation type holds metadata for a memory manager client, including a
- * name for the type and various properties that may affect how much memory
- * such a client will get, whether it always gets some, etc.
- */
-
-#pragma once
-
-#include <string>
-#include <memory>
-
-namespace storage::framework {
-
-struct MemoryAllocationType {
- using UP = std::unique_ptr<MemoryAllocationType>;
-
- enum Flags {
- NONE = 0x00,
- FORCE_ALLOCATE = 0x01,
- EXTERNAL_LOAD = 0x02,
- CACHE = 0x04
- };
-
- MemoryAllocationType()
- : _flags(NONE), _name("") {};
-
- MemoryAllocationType(const std::string& name, uint32_t flags = NONE)
- : _flags(flags), _name(name) {}
-
- const std::string& getName() const { return _name; }
- bool isAllocationsForced() const { return (_flags & FORCE_ALLOCATE); }
- bool isExternalLoad() const { return (_flags & EXTERNAL_LOAD); }
- bool isCache() const { return (_flags & CACHE); }
-
-private:
- uint32_t _flags;
- std::string _name;
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h b/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h
deleted file mode 100644
index a97038c2aab..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::framework::MemoryManagerInterface
- * \ingroup memory
- *
- * \brief Interface with functions clients need in order to use a memory manager
- *
- * This interface exist so clients can use a memory manager without actually
- * depending on the implementation of it.
- */
-
-#pragma once
-
-#include "memoryallocationtype.h"
-#include "memorytoken.h"
-#include "reducememoryusageinterface.h"
-#include <vector>
-
-namespace storage::framework {
-
-struct MemoryManagerInterface
-{
- typedef std::unique_ptr<MemoryManagerInterface> UP;
-
- virtual ~MemoryManagerInterface() {}
-
- virtual void setMaximumMemoryUsage(uint64_t max) = 0;
-
- /**
- * Registers the given allocation type by copying it, and returning
- * a reference to the copied object.
- */
- virtual const MemoryAllocationType&
- registerAllocationType(const MemoryAllocationType& type) = 0;
-
- /** Throws exception if failing to find type. */
- virtual const MemoryAllocationType&
- getAllocationType(const std::string& name) const = 0;
-
- /** Get an overview of all registration types. */
- virtual std::vector<const MemoryAllocationType*>
- getAllocationTypes() const = 0;
-
- /**
- * Decide how much to allocate for this request. Should be between min
- * and max, unless it's of a type that can be denied (such as external
- * requests), in which case we can also deny allocation by returning a null
- * token.
- */
- virtual MemoryToken::UP allocate(
- const MemoryAllocationType&,
- uint64_t min,
- uint64_t max,
- uint8_t priority,
- ReduceMemoryUsageInterface* = 0) = 0;
-
- /**
- * Utility function to see how much memory is available.
- */
- virtual uint64_t getMemorySizeFreeForPriority(uint8_t priority) const = 0;
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp b/storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp
deleted file mode 100644
index 3950810a010..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memorytoken.h"
-
-namespace storage {
-namespace framework {
-
-MemoryToken::~MemoryToken()
-{
-}
-
-} // framework
-} // storage
diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h b/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h
deleted file mode 100644
index cd389d64549..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::framework::MemoryToken
- * \ingroup memory
- *
- * \brief Token to keep by client for current allocations.
- *
- * This class is a token a memory manager client will get from the memory
- * manager when getting memory. It can be used to know how much you currently
- * have allocated, and through it you can request more or less memory.
- */
-
-#pragma once
-
-#include <memory>
-
-namespace storage::framework {
-
-class MemoryToken {
-protected:
-public:
- typedef std::unique_ptr<MemoryToken> UP;
- virtual ~MemoryToken();
-
- virtual uint64_t getSize() const = 0;
- virtual bool resize(uint64_t min, uint64_t max) = 0;
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h b/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h
deleted file mode 100644
index cd50d33015c..00000000000
--- a/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * \class storage::framework::ReduceMemoryUsageInterface
- * \ingroup memory
- *
- * \brief The manager can take memory back when needed using this interface.
- *
- * Some memory users, typically caches, wants to use all available memory. But
- * to let them use all available memory, it must also be easy to take memory
- * back when needed for something else. An implementation of this interface can
- * be given on memory allocations to give the memory manager the ability to take
- * memory back when needed.
- */
-
-#pragma once
-
-namespace storage::framework {
-
-struct ReduceMemoryUsageInterface
-{
- virtual ~ReduceMemoryUsageInterface() {}
-
- /**
- * This callback is called when the memory manager want to reduce the usage
- * of the given memory token. Actual memory to be released should be
- * released in this function. The token itself will be adjusted by the
- * memory manager though. The memory manager may keep a lock through this
- * call, so no memory manager calls should be made inside this callback.
- *
- * It is recommended that you actually release at least as many bytes as
- * requested. Though currently it is allowed to reduce less or refuse, but
- * this might mean that some higher priority task does not get the memory it
- * needs.
- *
- * @param reduceBy Always in the range 0 < reduceBy <= token.size()
- * @return The amount of memory no longer used.
- */
- virtual uint64_t reduceMemoryConsumption(const MemoryToken&, uint64_t reduceBy) = 0;
-};
-
-}
diff --git a/storageframework/src/vespa/storageframework/storageframework.h b/storageframework/src/vespa/storageframework/storageframework.h
index f9847d5fd12..4ab36fa099f 100644
--- a/storageframework/src/vespa/storageframework/storageframework.h
+++ b/storageframework/src/vespa/storageframework/storageframework.h
@@ -7,7 +7,6 @@
#include <vespa/storageframework/generic/clock/clock.h>
#include <vespa/storageframework/generic/clock/timer.h>
#include <vespa/storageframework/generic/component/component.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
#include <vespa/storageframework/generic/metric/metricupdatehook.h>
#include <vespa/storageframework/generic/status/htmlstatusreporter.h>
#include <vespa/storageframework/generic/status/statusreportermap.h>