Merge pull request #21120 from vespa-engine/revert-21119-revert-21118-balder/wire-in-mallopt-control-of-vespamalloc-skeletonv7.541.24

Revert "Revert "Wire in mallopt(in param, int value) interface in vespamalloc and ver…""

10 files changed, 111 insertions, 12 deletions

diff --git a/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp b/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp
index 79777cdd53f..348e9bbd503 100644
--- a/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp
+++ b/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/shutdownguard.h>
+#include <vespa/vespalib/util/malloc_mmap_guard.h>
 #include <thread>
 #include <unistd.h>
 #include <sys/wait.h>
@@ -8,12 +9,8 @@
 
 using namespace vespalib;
 
-TEST_SETUP(Test);
-
-int
-Test::Main()
+TEST("test shutdown guard")
 {
-    TEST_INIT("shutdownguard_test");
     {
         ShutdownGuard farFuture(1000000s);
         std::this_thread::sleep_for(20ms);
@@ -37,5 +34,10 @@ Test::Main()
         }
         EXPECT_TRUE(i < 800);
     }
-    TEST_DONE();
 }
+
+TEST("test malloc mmap guard") {
+    MallocMmapGuard guard(0x100000);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt
index 2b52e9e167f..e69dd36d6f5 100644
--- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt
+++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt
@@ -5,6 +5,7 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT
     bits.cpp
     clock.cpp
     crc.cpp
+    document_runnable.cpp
     doom.cpp
     foregroundtaskexecutor.cpp
     growablebytebuffer.cpp
@@ -12,10 +13,10 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT
     jsonexception.cpp
     jsonstream.cpp
     jsonwriter.cpp
+    malloc_mmap_guard.cpp
     process_memory_stats.cpp
     programoptions.cpp
     programoptions_testutils.cpp
-    document_runnable.cpp
     rusage.cpp
     sequencedtaskexecutor.cpp
     sequencedtaskexecutorobserver.cpp
diff --git a/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp b/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp
new file mode 100644
index 00000000000..0ced160fda2
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp
@@ -0,0 +1,23 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "malloc_mmap_guard.h"
+#include <vespa/vespalib/util/size_literals.h>
+#include <malloc.h>
+#include <limits>
+#include <cassert>
+
+namespace vespalib {
+
+MallocMmapGuard::MallocMmapGuard(size_t mmapLimit) :
+    _threadId(std::this_thread::get_id())
+{
+    int limit = mmapLimit <= std::numeric_limits<int>::max() ? mmapLimit : std::numeric_limits<int>::max();
+    mallopt(M_MMAP_THRESHOLD, limit);
+}
+
+MallocMmapGuard::~MallocMmapGuard()
+{
+    assert(_threadId == std::this_thread::get_id());
+    mallopt(M_MMAP_THRESHOLD, 1_Gi);
+}
+
+}
diff --git a/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h b/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h
new file mode 100644
index 00000000000..03e6d38c03c
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h
@@ -0,0 +1,28 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <thread>
+
+namespace vespalib {
+
+/**
+ * Provides a hint to malloc implementation that all allocations in the scope of this guard
+ * will use mmap directly for allocation larger than the given limit.
+ * NB !! Note that guards can not be nested. Intention is to use around third party libraries where
+ * you do not control allocation yourself.
+ * The effect is implementation dependent. vespamalloc applies this only for the calling thread.
+ **/
+class MallocMmapGuard
+{
+public:
+    MallocMmapGuard(size_t mmapLimit);
+    MallocMmapGuard(const MallocMmapGuard &) = delete;
+    MallocMmapGuard & operator=(const MallocMmapGuard &) = delete;
+    MallocMmapGuard(MallocMmapGuard &&) = delete;
+    MallocMmapGuard & operator=(MallocMmapGuard &&) = delete;
+    ~MallocMmapGuard();
+private:
+    std::thread::id _threadId;
+};
+
+} // namespace vespalib
diff --git a/vespamalloc/src/tests/stacktrace/stacktrace.cpp b/vespamalloc/src/tests/stacktrace/stacktrace.cpp
index b28a9653d27..2f0d2eb2277 100644
--- a/vespamalloc/src/tests/stacktrace/stacktrace.cpp
+++ b/vespamalloc/src/tests/stacktrace/stacktrace.cpp
@@ -1,6 +1,5 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <cstdlib>
-#include <cstdio>
 #include <pthread.h>
 #include <dlfcn.h>
 #include <cassert>
diff --git a/vespamalloc/src/tests/test1/new_test.cpp b/vespamalloc/src/tests/test1/new_test.cpp
index 5230869145d..0723f8cca85 100644
--- a/vespamalloc/src/tests/test1/new_test.cpp
+++ b/vespamalloc/src/tests/test1/new_test.cpp
@@ -134,14 +134,28 @@ void verify_vespamalloc_usable_size() {
     }
 }
 
+enum class MallocLibrary { UNKNOWN, VESPA_MALLOC, VESPA_MALLOC_D};
+
+MallocLibrary
+detectLibrary() {
+    if (dlsym(RTLD_NEXT, "is_vespamallocd") != nullptr) {
+        // Debug variants will never have more memory available as there is pre/postamble for error detection.
+        return MallocLibrary::VESPA_MALLOC_D;
+    } else if (dlsym(RTLD_NEXT, "is_vespamalloc") != nullptr) {
+        return MallocLibrary::VESPA_MALLOC;
+    }
+    return MallocLibrary::UNKNOWN;
+}
+
 TEST("verify malloc_usable_size is sane") {
     constexpr size_t SZ = 33;
     std::unique_ptr<char[]> buf = std::make_unique<char[]>(SZ);
     size_t usable_size = malloc_usable_size(buf.get());
-    if (dlsym(RTLD_NEXT, "is_vespamallocd") != nullptr) {
+    MallocLibrary env = detectLibrary();
+    if (env == MallocLibrary::VESPA_MALLOC_D) {
         // Debug variants will never have more memory available as there is pre/postamble for error detection.
         EXPECT_EQUAL(SZ, usable_size);
-    } else if (dlsym(RTLD_NEXT, "is_vespamalloc") != nullptr) {
+    } else if (env == MallocLibrary::VESPA_MALLOC) {
         // Normal production vespamalloc will round up
         EXPECT_EQUAL(64u, usable_size);
         verify_vespamalloc_usable_size();
@@ -151,5 +165,13 @@ TEST("verify malloc_usable_size is sane") {
     }
 }
 
+TEST("verify mallopt") {
+    MallocLibrary env = detectLibrary();
+    if (env == MallocLibrary::UNKNOWN) return;
+    EXPECT_EQUAL(0, mallopt(M_MMAP_MAX, 0x1000000));
+    EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 0x1000000));
+    EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, -1));
+}
+
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.h b/vespamalloc/src/vespamalloc/malloc/malloc.h
index 8e0a39e944b..449dce29896 100644
--- a/vespamalloc/src/vespamalloc/malloc/malloc.h
+++ b/vespamalloc/src/vespamalloc/malloc/malloc.h
@@ -25,6 +25,7 @@ public:
     }
     size_t getMaxNumThreads() const override { return _threadList.getMaxNumThreads(); }
 
+    int mallopt(int param, int value);
     void *malloc(size_t sz);
     void *malloc(size_t sz, std::align_val_t);
     void *realloc(void *oldPtr, size_t sz);
@@ -149,6 +150,11 @@ void MemoryManager<MemBlockPtrT, ThreadListT>::info(FILE * os, size_t level)
 }
 
 template <typename MemBlockPtrT, typename ThreadListT>
+int MemoryManager<MemBlockPtrT, ThreadListT>::mallopt(int param, int value) {
+    return _threadList.getCurrent().mallopt(param, value);
+}
+
+template <typename MemBlockPtrT, typename ThreadListT>
 void * MemoryManager<MemBlockPtrT, ThreadListT>::malloc(size_t sz)
 {
     MemBlockPtrT mem;
@@ -205,7 +211,7 @@ void MemoryManager<MemBlockPtrT, ThreadListT>::freeSC(void *ptr, SizeClassT sc)
 template <typename MemBlockPtrT, typename ThreadListT>
 void * MemoryManager<MemBlockPtrT, ThreadListT>::realloc(void *oldPtr, size_t sz)
 {
-    void *ptr(NULL);
+    void *ptr(nullptr);
     if (oldPtr) {
         MemBlockPtrT mem(oldPtr);
         mem.readjustAlignment(_segment);
diff --git a/vespamalloc/src/vespamalloc/malloc/overload.h b/vespamalloc/src/vespamalloc/malloc/overload.h
index 69d95ef5cdc..7d9c2b9c72e 100644
--- a/vespamalloc/src/vespamalloc/malloc/overload.h
+++ b/vespamalloc/src/vespamalloc/malloc/overload.h
@@ -4,7 +4,7 @@
 #include <dlfcn.h>
 #include <errno.h>
 #include <new>
-#include <stdlib.h>
+#include <cstdlib>
 #include <malloc.h>
 
 class CreateAllocator
@@ -140,6 +140,11 @@ struct mallinfo mallinfo() __THROW {
 }
 #endif
 
+int mallopt(int param, int value) throw() __attribute((visibility("default")));
+int mallopt(int param, int value) throw() {
+    return vespamalloc::createAllocator()->mallopt(param, value);
+}
+
 void * malloc(size_t sz) __attribute((visibility("default")));
 void * malloc(size_t sz) {
     return vespamalloc::createAllocator()->malloc(sz);
diff --git a/vespamalloc/src/vespamalloc/malloc/threadpool.h b/vespamalloc/src/vespamalloc/malloc/threadpool.h
index ac4f5bb8551..ec89079a415 100644
--- a/vespamalloc/src/vespamalloc/malloc/threadpool.h
+++ b/vespamalloc/src/vespamalloc/malloc/threadpool.h
@@ -19,6 +19,7 @@ public:
     void setPool(AllocPool & pool) {
         _allocPool = & pool;
     }
+    int mallopt(int param, int value);
     void malloc(size_t sz, MemBlockPtrT & mem);
     void free(MemBlockPtrT mem, SizeClassT sc);
 
@@ -65,6 +66,7 @@ private:
     static constexpr bool alwaysReuse(SizeClassT sc) { return sc > ALWAYS_REUSE_SC_LIMIT; }
 
     AllocPool   * _allocPool;
+    ssize_t       _mmapLimit;
     AllocFree     _memList[NUM_SIZE_CLASSES];
     ThreadStatT   _stat[NUM_SIZE_CLASSES];
     uint32_t      _threadId;
diff --git a/vespamalloc/src/vespamalloc/malloc/threadpool.hpp b/vespamalloc/src/vespamalloc/malloc/threadpool.hpp
index 7e86c3f691a..e9b9fabebdc 100644
--- a/vespamalloc/src/vespamalloc/malloc/threadpool.hpp
+++ b/vespamalloc/src/vespamalloc/malloc/threadpool.hpp
@@ -2,6 +2,7 @@
 #pragma once
 
 #include <vespamalloc/malloc/threadpool.h>
+#include <malloc.h>
 
 namespace vespamalloc {
 
@@ -85,6 +86,7 @@ mallocHelper(size_t exactSize,
 template <typename MemBlockPtrT, typename ThreadStatT >
 ThreadPoolT<MemBlockPtrT, ThreadStatT>::ThreadPoolT() :
     _allocPool(nullptr),
+    _mmapLimit(0x40000000),
     _threadId(0),
     _osThreadId(0)
 {
@@ -94,6 +96,15 @@ template <typename MemBlockPtrT, typename ThreadStatT >
 ThreadPoolT<MemBlockPtrT, ThreadStatT>::~ThreadPoolT() = default;
 
 template <typename MemBlockPtrT, typename ThreadStatT >
+int ThreadPoolT<MemBlockPtrT, ThreadStatT>::mallopt(int param, int value) {
+    if (param == M_MMAP_THRESHOLD) {
+        _mmapLimit = value;
+        return 1;
+    }
+    return 0;
+}
+
+template <typename MemBlockPtrT, typename ThreadStatT >
 void ThreadPoolT<MemBlockPtrT, ThreadStatT>::malloc(size_t sz, MemBlockPtrT & mem)
 {
     SizeClassT sc = MemBlockPtrT::sizeClass(sz);