Calculate transient memory usage for load of attribute vector.

23 files changed, 320 insertions, 18 deletions

diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
index 1f0f3566b1d..6eaa1bd373a 100644
--- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
@@ -3,11 +3,14 @@
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/test/make_bucket_space.h>
 #include <vespa/fastos/thread.h>
+#include <vespa/config-attributes.h>
+#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h>
 #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h>
 #include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
 #include <vespa/searchcore/proton/bucketdb/bucket_create_notifier.h>
 #include <vespa/searchcore/proton/common/doctypename.h>
 #include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h>
 #include <vespa/searchcore/proton/documentmetastore/operation_listener.h>
 #include <vespa/searchcore/proton/feedoperation/moveoperation.h>
 #include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h>
@@ -60,6 +63,7 @@ using storage::spi::Timestamp;
 using vespalib::Slime;
 using vespalib::makeClosure;
 using vespalib::makeTask;
+using vespa::config::search::AttributesConfigBuilder;
 
 using BlockedReason = IBlockableMaintenanceJob::BlockedReason;
 
@@ -885,6 +889,8 @@ MaintenanceControllerFixture::injectMaintenanceJobs()
                                             _jobTrackers, *this,
                                             _readyAttributeManager,
                                             _notReadyAttributeManager,
+                                            std::make_unique<const AttributeConfigInspector>(AttributesConfigBuilder()),
+                                            std::make_shared<TransientMemoryUsageProvider>(),
                                             _attributeUsageFilter);
     }
 }
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
index 550d0a1c4c4..e0b0cd9f0fd 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
@@ -5,6 +5,7 @@ vespa_add_library(searchcore_attribute STATIC
     attribute_aspect_delayer.cpp
     attribute_collection_spec_factory.cpp
     attribute_collection_spec.cpp
+    attribute_config_inspector.cpp
     attribute_directory.cpp
     attribute_factory.cpp
     attribute_initializer.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp
new file mode 100644
index 00000000000..d950b5e8ed3
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp
@@ -0,0 +1,30 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "attribute_config_inspector.h"
+#include <vespa/searchlib/attribute/configconverter.h>
+#include <vespa/config-attributes.h>
+#include <vespa/vespalib/stllike/hash_map.hpp>
+
+using search::attribute::ConfigConverter;
+
+namespace proton {
+
+AttributeConfigInspector::AttributeConfigInspector(const AttributesConfig& config)
+    : _hash()
+{
+    for (auto& attr : config.attribute) {
+        auto res = _hash.insert(std::make_pair(attr.name, ConfigConverter::convert(attr)));
+        assert(res.second);
+    }
+}
+
+AttributeConfigInspector::~AttributeConfigInspector() = default;
+
+const search::attribute::Config*
+AttributeConfigInspector::get_config(const vespalib::string& name) const
+{
+    auto itr = _hash.find(name);
+    return (itr != _hash.end()) ? &itr->second : nullptr;
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h
new file mode 100644
index 00000000000..8952b34c6ff
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h
@@ -0,0 +1,26 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/searchcommon/attribute/config.h>
+
+namespace vespa::config::search::internal { class InternalAttributesType; }
+
+namespace proton {
+
+/*
+ * Class used to find attribute config given attribute name based on config
+ * from config server.
+ */
+class AttributeConfigInspector {
+    vespalib::hash_map<vespalib::string, search::attribute::Config> _hash;
+public:
+    using AttributesConfig = const vespa::config::search::internal::InternalAttributesType;
+    AttributeConfigInspector(const AttributesConfig& config);
+    ~AttributeConfigInspector();
+    const search::attribute::Config* get_config(const vespalib::string& name) const;
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp
index 39235fb8d00..43c9e52315b 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp
@@ -2,28 +2,35 @@
 
 #include "attribute_usage_sampler_context.h"
 #include "attribute_usage_filter.h"
+#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h>
 
 namespace proton {
 
-AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter &filter)
+AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter &filter, std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider)
     : _usage(),
+      _transient_memory_usage(0u),
       _lock(),
-      _filter(filter)
+      _filter(filter),
+      _attribute_config_inspector(std::move(attribute_config_inspector)),
+      _transient_memory_usage_provider(std::move(transient_memory_usage_provider))
 {
 }
 
 AttributeUsageSamplerContext::~AttributeUsageSamplerContext()
 {
     _filter.setAttributeStats(_usage);
+    _transient_memory_usage_provider->set_transient_memory_usage(_transient_memory_usage);
 }
 
 void
 AttributeUsageSamplerContext::merge(const search::AddressSpaceUsage &usage,
+                                    size_t transient_memory_usage,
                                     const vespalib::string &attributeName,
                                     const vespalib::string &subDbName)
 {
     Guard guard(_lock);
     _usage.merge(usage, attributeName, subDbName);
+    _transient_memory_usage = std::max(_transient_memory_usage, transient_memory_usage);
 }
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h
index 21e4ff6a90f..b5693aef1bc 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h
@@ -8,6 +8,8 @@
 namespace proton {
 
 class AttributeUsageFilter;
+class AttributeConfigInspector;
+class TransientMemoryUsageProvider;
 
 /*
  * Context for sampling attribute usage stats.  When instance is
@@ -19,16 +21,22 @@ class AttributeUsageSamplerContext
     using Guard = std::lock_guard<Mutex>;
 
     AttributeUsageStats _usage;
+    size_t _transient_memory_usage;
     Mutex _lock;
     AttributeUsageFilter &_filter;
+    std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector;
+    std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider;
 public:
-    AttributeUsageSamplerContext(AttributeUsageFilter &filter);
+    AttributeUsageSamplerContext(AttributeUsageFilter &filter, std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider);
     ~AttributeUsageSamplerContext();
     void merge(const search::AddressSpaceUsage &usage,
+               size_t transient_memory_usage,
                const vespalib::string &attributeName,
                const vespalib::string &subDbName);
+    const AttributeConfigInspector& get_attribute_config_inspector() const { return *_attribute_config_inspector; }
     const AttributeUsageStats &
     getUsage() const { return _usage; }
+    size_t get_transient_memory_usage() const { return _transient_memory_usage; }
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp
index a1a8409e93b..f8594a4477b 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp
@@ -2,10 +2,49 @@
 
 #include "attribute_usage_sampler_functor.h"
 #include "attribute_usage_sampler_context.h"
+#include "attribute_config_inspector.h"
 #include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/attribute/loadedenumvalue.h>
+#include <vespa/searchlib/attribute/loadedvalue.h>
+
+using search::attribute::BasicType;
 
 namespace proton {
 
+namespace {
+
+size_t
+get_transient_memory_usage(const search::attribute::Config& old_config,
+                           const search::attribute::Config& current_config,
+                           uint64_t total_value_count)
+{
+    if (current_config.fastSearch()) {
+        if (old_config.fastSearch()) {
+            return sizeof(search::attribute::LoadedEnumAttribute) * total_value_count;
+        } else {
+            switch (old_config.basicType().type()) {
+            case BasicType::Type::INT8:
+                return sizeof(search::attribute::LoadedValue<int8_t>) * total_value_count;
+            case BasicType::Type::INT16:
+                return sizeof(search::attribute::LoadedValue<int16_t>) * total_value_count;
+            case BasicType::Type::INT32:
+                return sizeof(search::attribute::LoadedValue<int32_t>) * total_value_count;
+            case BasicType::Type::INT64:
+                return sizeof(search::attribute::LoadedValue<int64_t>) * total_value_count;
+            case BasicType::Type::FLOAT:
+                return sizeof(search::attribute::LoadedValue<float>) * total_value_count;
+            case BasicType::Type::DOUBLE:
+                return sizeof(search::attribute::LoadedValue<double>) * total_value_count;
+            default:
+                ;
+            }
+        }
+    }
+    return 0u;
+}
+
+}
+
 AttributeUsageSamplerFunctor::AttributeUsageSamplerFunctor(
         std::shared_ptr<AttributeUsageSamplerContext> samplerContext,
         const std::string &subDbName)
@@ -23,7 +62,15 @@ AttributeUsageSamplerFunctor::operator()(const search::attribute::IAttributeVect
     const auto & attributeVector = dynamic_cast<const search::AttributeVector &>(iAttributeVector);
     search::AddressSpaceUsage usage = attributeVector.getAddressSpaceUsage();
     vespalib::string attributeName = attributeVector.getName();
-    _samplerContext->merge(usage, attributeName, _subDbName);
+    size_t transient_memory_usage = 0;
+    auto& old_config = attributeVector.getConfig();
+    auto* current_config = _samplerContext->get_attribute_config_inspector().get_config(attributeName);
+    if (current_config == nullptr) {
+        current_config = &old_config;
+    }
+    uint64_t total_value_count = attributeVector.getStatus().getNumValues();
+    transient_memory_usage = get_transient_memory_usage(old_config, *current_config, total_value_count);
+    _samplerContext->merge(usage, transient_memory_usage, attributeName, _subDbName);
 }
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
index f3303f36199..7a9426ae716 100644
--- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
@@ -20,6 +20,7 @@ vespa_add_library(searchcore_pcommon STATIC
     selectpruner.cpp
     state_reporter_utils.cpp
     statusreport.cpp
+    transient_memory_usage_provider.cpp
     DEPENDS
     searchcore_proton_metrics
     searchcore_fconfig
diff --git a/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h
new file mode 100644
index 00000000000..e00ccfd3c27
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h
@@ -0,0 +1,19 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <cstddef>
+
+namespace proton {
+
+/*
+ * Interface class providing transient memory usage, e.g. extra memory needed
+ * for loading or saving an attribute vector.
+ */
+class ITransientMemoryUsageProvider {
+public:
+    virtual ~ITransientMemoryUsageProvider() = default;
+    virtual size_t get_transient_memory_usage() const = 0;
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp
new file mode 100644
index 00000000000..0752af29a0a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp
@@ -0,0 +1,27 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "transient_memory_usage_provider.h"
+
+namespace proton {
+
+TransientMemoryUsageProvider::TransientMemoryUsageProvider()
+    : ITransientMemoryUsageProvider(),
+      _transient_memory_usage(0u)
+{
+}
+
+TransientMemoryUsageProvider::~TransientMemoryUsageProvider() = default;
+
+size_t
+TransientMemoryUsageProvider::get_transient_memory_usage() const
+{
+    return _transient_memory_usage.load(std::memory_order_relaxed);
+}
+
+void
+TransientMemoryUsageProvider::set_transient_memory_usage(size_t transient_memory_usage)
+{
+    _transient_memory_usage.store(transient_memory_usage, std::memory_order_relaxed);
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h
new file mode 100644
index 00000000000..42c6933677a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h
@@ -0,0 +1,24 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_transient_memory_usage_provider.h"
+
+#include <atomic>
+
+namespace proton {
+
+/*
+ * Class providing transient memory usage, e.g. extra memory needed
+ * for loading or saving an attribute vector.
+ */
+class TransientMemoryUsageProvider : public ITransientMemoryUsageProvider {
+    std::atomic<size_t> _transient_memory_usage;
+public:
+    TransientMemoryUsageProvider();
+    virtual ~TransientMemoryUsageProvider();
+    size_t get_transient_memory_usage() const override;
+    void set_transient_memory_usage(size_t transient_memory_usage);
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
index 3191d82bf11..daefaf2883e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
@@ -156,6 +156,7 @@ DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo)
       _hwInfo(hwInfo),
       _memoryStats(),
       _diskUsedSizeBytes(),
+      _transient_memory_usage(0u),
       _config(),
       _state(),
       _acceptWrite(true),
@@ -182,6 +183,13 @@ DiskMemUsageFilter::setDiskUsedSize(uint64_t diskUsedSizeBytes)
 }
 
 void
+DiskMemUsageFilter::set_transient_memory_usage(size_t transient_memory_usage)
+{
+    Guard guard(_lock);
+    _transient_memory_usage = transient_memory_usage;
+}
+
+void
 DiskMemUsageFilter::setConfig(Config config_in)
 {
     Guard guard(_lock);
@@ -203,6 +211,13 @@ DiskMemUsageFilter::getDiskUsedSize() const
     return _diskUsedSizeBytes;
 }
 
+size_t
+DiskMemUsageFilter::get_transient_memory_usage() const
+{
+    Guard guard(_lock);
+    return _transient_memory_usage;
+}
+
 DiskMemUsageFilter::Config
 DiskMemUsageFilter::getConfig() const
 {
@@ -230,7 +245,6 @@ DiskMemUsageFilter::getAcceptState() const
     return _state;
 }
 
-
 void
 DiskMemUsageFilter::addDiskMemUsageListener(IDiskMemUsageListener *listener)
 {
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
index afb8c3d2c7a..f3009c2ce97 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
@@ -43,6 +43,7 @@ private:
     HwInfo                       _hwInfo;
     vespalib::ProcessMemoryStats _memoryStats;
     uint64_t                     _diskUsedSizeBytes;
+    size_t                       _transient_memory_usage;
     Config                       _config;
     State                        _state;
     std::atomic<bool>            _acceptWrite;
@@ -59,9 +60,11 @@ public:
     ~DiskMemUsageFilter() override;
     void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in);
     void setDiskUsedSize(uint64_t diskUsedSizeBytes);
+    void set_transient_memory_usage(size_t transient_memory_usage);
     void setConfig(Config config);
     vespalib::ProcessMemoryStats getMemoryStats() const;
     uint64_t getDiskUsedSize() const;
+    size_t get_transient_memory_usage() const;
     Config getConfig() const;
     const HwInfo &getHwInfo() const { return _hwInfo; }
     DiskMemUsageState usageState() const;
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
index a3fb1244c45..f7f505cd754 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
@@ -3,6 +3,7 @@
 #include "disk_mem_usage_sampler.h"
 #include <vespa/vespalib/util/scheduledexecutor.h>
 #include <vespa/vespalib/util/lambdatask.h>
+#include <vespa/searchcore/proton/common/i_transient_memory_usage_provider.h>
 #include <filesystem>
 
 using vespalib::makeLambdaTask;
@@ -14,7 +15,9 @@ DiskMemUsageSampler::DiskMemUsageSampler(const std::string &path_in, const Confi
       _path(path_in),
       _sampleInterval(60s),
       _lastSampleTime(vespalib::steady_clock::now()),
-      _periodicTimer()
+      _periodicTimer(),
+      _lock(),
+      _transient_memory_usage_providers()
 {
     setConfig(config);
 }
@@ -49,6 +52,7 @@ DiskMemUsageSampler::sampleUsage()
 {
     sampleMemoryUsage();
     sampleDiskUsage();
+    sample_transient_memory_usage();
 }
 
 namespace {
@@ -118,4 +122,37 @@ DiskMemUsageSampler::sampleMemoryUsage()
     _filter.setMemoryStats(vespalib::ProcessMemoryStats::create());
 }
 
+void
+DiskMemUsageSampler::sample_transient_memory_usage()
+{
+    size_t max_transient_memory_usage = 0;
+    {
+        std::lock_guard<std::mutex> guard(_lock);
+        for (auto provider : _transient_memory_usage_providers) {
+            auto transient_memory_usage = provider->get_transient_memory_usage();
+            max_transient_memory_usage = std::max(max_transient_memory_usage, transient_memory_usage);
+        }
+    }
+    _filter.set_transient_memory_usage(max_transient_memory_usage);
+}
+
+void
+DiskMemUsageSampler::add_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider)
+{
+    std::lock_guard<std::mutex> guard(_lock);
+    _transient_memory_usage_providers.push_back(provider);
+}
+
+void
+DiskMemUsageSampler::remove_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider)
+{
+    std::lock_guard<std::mutex> guard(_lock);
+    for (auto itr = _transient_memory_usage_providers.begin(); itr != _transient_memory_usage_providers.end(); ++itr) {
+        if (*itr == provider) {
+            _transient_memory_usage_providers.erase(itr);
+            break;
+        }
+    }
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
index bbfc95ff82c..38e80a6cec1 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
@@ -9,6 +9,8 @@ namespace vespalib { class ScheduledExecutor; }
 
 namespace proton {
 
+class ITransientMemoryUsageProvider;
+
 /*
  * Class to sample disk and memory usage used for filtering write operations.
  */
@@ -18,10 +20,13 @@ class DiskMemUsageSampler {
     vespalib::duration    _sampleInterval;
     vespalib::steady_time _lastSampleTime;
     std::unique_ptr<vespalib::ScheduledExecutor> _periodicTimer;
+    std::mutex            _lock;
+    std::vector<std::shared_ptr<const ITransientMemoryUsageProvider>> _transient_memory_usage_providers;
 
     void sampleUsage();
     void sampleDiskUsage();
     void sampleMemoryUsage();
+    void sample_transient_memory_usage();
 public:
     struct Config {
         DiskMemUsageFilter::Config filterConfig;
@@ -55,6 +60,8 @@ public:
 
     const DiskMemUsageFilter &writeFilter() const { return _filter; }
     IDiskMemUsageNotifier &notifier() { return _filter; }
+    void add_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider);
+    void remove_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider);
 };
 
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 55f95ce0518..7567aa9c322 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -13,10 +13,12 @@
 #include "maintenance_jobs_injector.h"
 #include "reconfig_params.h"
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h>
 #include <vespa/searchcore/proton/attribute/attribute_writer.h>
 #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
 #include <vespa/searchcore/proton/common/eventlogger.h>
 #include <vespa/searchcore/proton/common/statusreport.h>
+#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h>
 #include <vespa/searchcore/proton/docsummary/isummarymanager.h>
 #include <vespa/searchcore/proton/feedoperation/noopoperation.h>
 #include <vespa/searchcore/proton/index/index_writer.h>
@@ -163,6 +165,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
       _state(),
       _dmUsageForwarder(_writeService.master()),
       _writeFilter(),
+      _transient_memory_usage_provider(std::make_shared<TransientMemoryUsageProvider>()),
       _feedHandler(_writeService, tlsSpec, docTypeName, _state, *this, _writeFilter, *this, tlsDirectWriter),
       _subDBs(*this, *this, _feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext,
               metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir,
@@ -929,7 +932,7 @@ DocumentDB::hasDocument(const document::DocumentId &id)
 }
 
 void
-DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config)
+DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector)
 {
     // Called by executor thread
     _maintenanceController.killJobs();
@@ -954,6 +957,8 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config)
             _visibility,  // ICommitable
             _subDBs.getReadySubDB()->getAttributeManager(),
             _subDBs.getNotReadySubDB()->getAttributeManager(),
+            std::move(attribute_config_inspector),
+            _transient_memory_usage_provider,
             _writeFilter);
 }
 
@@ -963,18 +968,21 @@ DocumentDB::performStartMaintenance()
     // Called by executor thread
     // Only start once, after replay done
 
-    DocumentDBMaintenanceConfig::SP maintenanceConfig;
+    std::shared_ptr<DocumentDBConfig> activeConfig;
     {
         lock_guard guard(_configMutex);
         if (_state.getClosed())
             return;
-        assert(_activeConfigSnapshot);
-        maintenanceConfig = _activeConfigSnapshot->getMaintenanceConfigSP();
+        activeConfig = _activeConfigSnapshot;
     }
+    assert(activeConfig);
     if (_maintenanceController.getStopping()) {
         return;
     }
-    injectMaintenanceJobs(*maintenanceConfig);
+    auto maintenanceConfig = activeConfig->getMaintenanceConfigSP();
+    const auto &attributes_config = activeConfig->getAttributesConfig();
+    auto attribute_config_inspector = std::make_unique<AttributeConfigInspector>(attributes_config);
+    injectMaintenanceJobs(*maintenanceConfig, std::move(attribute_config_inspector));
     _maintenanceController.start(maintenanceConfig);
 }
 
@@ -992,10 +1000,12 @@ DocumentDB::forwardMaintenanceConfig()
     assert(activeConfig);
     DocumentDBMaintenanceConfig::SP
         maintenanceConfig(activeConfig->getMaintenanceConfigSP());
+    const auto &attributes_config = activeConfig->getAttributesConfig();
+    auto attribute_config_inspector = std::make_unique<AttributeConfigInspector>(attributes_config);
     if (!_state.getClosed()) {
         if (_maintenanceController.getStarted() &&
             !_maintenanceController.getStopping()) {
-            injectMaintenanceJobs(*maintenanceConfig);
+            injectMaintenanceJobs(*maintenanceConfig, std::move(attribute_config_inspector));
         }
         _maintenanceController.newConfig(maintenanceConfig);
     }
@@ -1086,4 +1096,10 @@ DocumentDB::getDistributionKey() const
     return _owner.getDistributionKey();
 }
 
+std::shared_ptr<const ITransientMemoryUsageProvider>
+DocumentDB::transient_memory_usage_provider()
+{
+    return _transient_memory_usage_provider;
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index 917d753683a..33dfa85b628 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -44,10 +44,13 @@ namespace search {
 namespace vespa::config::search::core::internal { class InternalProtonType; }
 
 namespace proton {
+class AttributeConfigInspector;
 class IDocumentDBOwner;
+class ITransientMemoryUsageProvider;
 struct MetricsWireService;
 class StatusReport;
 class ExecutorThreadingServiceStats;
+class TransientMemoryUsageProvider;
 
 namespace matching { class SessionManager; }
 
@@ -132,6 +135,7 @@ private:
     DDBState                      _state;
     DiskMemUsageForwarder         _dmUsageForwarder;
     AttributeUsageFilter          _writeFilter;
+    std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider;
     FeedHandler                   _feedHandler;
 
     DocumentSubDBCollection       _subDBs;
@@ -412,7 +416,7 @@ public:
     /**
      * Implements IFeedHandlerOwner
      **/
-    void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config);
+    void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector);
     void performStartMaintenance();
     void stopMaintenance();
     void forwardMaintenanceConfig();
@@ -435,6 +439,7 @@ public:
     void enterOnlineState();
     void waitForOnlineState();
     IDiskMemUsageListener *diskMemUsageListener() { return &_dmUsageForwarder; }
+    std::shared_ptr<const ITransientMemoryUsageProvider> transient_memory_usage_provider();
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp
index 483497eb008..d69f0365fc0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp
@@ -9,6 +9,7 @@
 #include "prune_session_cache_job.h"
 #include "pruneremoveddocumentsjob.h"
 #include "sample_attribute_usage_job.h"
+#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h>
 
 using vespalib::system_clock;
 
@@ -99,6 +100,8 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller,
                                     ICommitable &commit,
                                     IAttributeManagerSP readyAttributeManager,
                                     IAttributeManagerSP notReadyAttributeManager,
+                                    std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector,
+                                    std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider,
                                     AttributeUsageFilter &attributeUsageFilter) {
     controller.registerJobInMasterThread(std::make_unique<HeartBeatJob>(hbHandler, config.getHeartBeatConfig()));
     controller.registerJobInDefaultPool(std::make_unique<PruneSessionCacheJob>(scPruner, config.getSessionCachePruneInterval()));
@@ -123,7 +126,9 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller,
                                                   notReadyAttributeManager,
                                                   attributeUsageFilter,
                                                   docTypeName,
-                                                  config.getAttributeUsageSampleInterval()));
+                                                  config.getAttributeUsageSampleInterval(),
+                                                  std::move(attribute_config_inspector),
+                                                  transient_memory_usage_provider));
 }
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h
index 94203e144dc..f0859335919 100644
--- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h
@@ -12,6 +12,7 @@
 
 namespace proton {
 
+class AttributeConfigInspector;
 class IPruneRemovedDocumentsHandler;
 struct IDocumentMoveHandler;
 class IBucketModifiedHandler;
@@ -21,6 +22,7 @@ struct IBucketStateCalculator;
 struct IAttributeManager;
 class AttributeUsageFilter;
 class IDiskMemUsageNotifier;
+class TransientMemoryUsageProvider;
 namespace bucketdb { class IBucketCreateNotifier; }
 
 /**
@@ -53,6 +55,8 @@ struct MaintenanceJobsInjector
                            ICommitable & commit,
                            IAttributeManagerSP readyAttributeManager,
                            IAttributeManagerSP notReadyAttributeManager,
+                           std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector,
+                           std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider,
                            AttributeUsageFilter &attributeUsageFilter);
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index c886b371064..cd9c1467da9 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -606,6 +606,7 @@ Proton::addDocumentDB(const document::DocumentType &docType,
     auto flushHandler = std::make_shared<FlushHandlerProxy>(ret);
     _flushEngine->putFlushHandler(docTypeName, flushHandler);
     _diskMemUsageSampler->notifier().addDiskMemUsageListener(ret->diskMemUsageListener());
+    _diskMemUsageSampler->add_transient_memory_usage_provider(ret->transient_memory_usage_provider());
     return ret;
 }
 
@@ -643,6 +644,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName)
     _metricsEngine->removeMetricsHook(old->getMetricsUpdateHook());
     _metricsEngine->removeDocumentDBMetrics(old->getMetrics());
     _diskMemUsageSampler->notifier().removeDiskMemUsageListener(old->diskMemUsageListener());
+    _diskMemUsageSampler->remove_transient_memory_usage_provider(old->transient_memory_usage_provider());
     // Caller should have removed & drained relevant timer tasks
     old->close();
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
index 6f1fd8b90e0..83c268fc755 100644
--- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
@@ -47,6 +47,8 @@ ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool
         memory.setDouble("utilization", usageState.memoryState().utilization());
         memory.setLong("physicalMemory", _usageFilter.getHwInfo().memory().sizeBytes());
         convertMemoryStatsToSlime(_usageFilter.getMemoryStats(), memory.setObject("stats"));
+        size_t transient_memory = _usageFilter.get_transient_memory_usage();
+        memory.setLong("transient", transient_memory);
     } else {
         object.setDouble("disk", usageState.diskState().usage());
         object.setDouble("memory", usageState.memoryState().usage());
diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp
index 1f5f29c7708..a6497966891 100644
--- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp
@@ -2,6 +2,7 @@
 
 #include "sample_attribute_usage_job.h"
 #include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
+#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h>
 #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h>
 #include <vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h>
 #include <vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h>
@@ -13,11 +14,15 @@ SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager,
                         IAttributeManagerSP notReadyAttributeManager,
                         AttributeUsageFilter &attributeUsageFilter,
                         const vespalib::string &docTypeName,
-                        vespalib::duration interval)
+                        vespalib::duration interval,
+                        std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector,
+                        std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider)
     : IMaintenanceJob("sample_attribute_usage." + docTypeName, vespalib::duration::zero(), interval),
       _readyAttributeManager(readyAttributeManager),
       _notReadyAttributeManager(notReadyAttributeManager),
-      _attributeUsageFilter(attributeUsageFilter)
+      _attributeUsageFilter(attributeUsageFilter),
+      _attribute_config_inspector(std::move(attribute_config_inspector)),
+      _transient_memory_usage_provider(std::move(transient_memory_usage_provider))
 {
 }
 
@@ -26,7 +31,7 @@ SampleAttributeUsageJob::~SampleAttributeUsageJob() = default;
 bool
 SampleAttributeUsageJob::run()
 {
-    auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter);
+    auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter, _attribute_config_inspector, _transient_memory_usage_provider);
     _readyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "ready"));
     _notReadyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "notready"));
     return true;
diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h
index 72a0bf1a665..6efbfb67c4c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h
+++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h
@@ -7,7 +7,9 @@
 namespace proton {
 
 struct IAttributeManager;
+class AttributeConfigInspector;
 class AttributeUsageFilter;
+class TransientMemoryUsageProvider;
 
 /**
  * Class used to sample attribute resource usage and pass aggregated
@@ -21,12 +23,16 @@ class SampleAttributeUsageJob : public IMaintenanceJob
     IAttributeManagerSP   _readyAttributeManager;
     IAttributeManagerSP   _notReadyAttributeManager;
     AttributeUsageFilter &_attributeUsageFilter;
+    std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector;
+    std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider;
 public:
     SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager,
                             IAttributeManagerSP notReadyAttributeManager,
                             AttributeUsageFilter &attributeUsageFilter,
                             const vespalib::string &docTypeName,
-                            vespalib::duration interval);
+                            vespalib::duration interval,
+                            std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector,
+                            std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider);
     ~SampleAttributeUsageJob() override;
 
     bool run() override;