Report transient resource usage for all attributes and document meta store.

20 files changed, 97 insertions, 17 deletions

diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
index 0e3445d0785..a66785cb567 100644
--- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -865,6 +865,15 @@ TEST_F("require that shrink flushtarget is handed over to new attribute manager"
     EXPECT_EQUAL(am1->getShrinker("a1"), am3->getShrinker("a1"));
 }
 
+TEST_F("transient resource usage is zero in steady state", Fixture)
+{
+    f.addAttribute("a1");
+    f.addAttribute("a2");
+    auto usage = f._m.get_transient_resource_usage();
+    EXPECT_EQUAL(0u, usage.disk());
+    EXPECT_EQUAL(0u, usage.memory());
+}
+
 TEST_MAIN()
 {
     std::filesystem::remove_all(std::filesystem::path(test_dir));
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index 12477469e04..43717fc724f 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -749,6 +749,13 @@ TEST_F("require that flush targets can be retrieved", SearchableFixture)
     EXPECT_TRUE(assertTarget("subdb.summary.shrink", FType::GC, FComponent::DOCUMENT_STORE, *targets[9]));
 }
 
+TEST_F("transient resource usage is zero in steady state", SearchableFixture)
+{
+    auto usage = f._subDb.get_transient_resource_usage();
+    EXPECT_EQUAL(0u, usage.disk());
+    EXPECT_EQUAL(0u, usage.memory());
+}
+
 TEST_F("require that only fast-access attributes are instantiated", FastAccessOnlyFixture)
 {
     std::vector<AttributeGuard> attrs;
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
index 939ae196de8..c03ae3b2f1f 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
@@ -650,4 +650,18 @@ AttributeManager::readable_attribute_vector(const string& name) const
     return _importedAttributes->get(name);
 }
 
+TransientResourceUsage
+AttributeManager::get_transient_resource_usage() const
+{
+    // Transient disk usage is measured as the total disk usage of all attribute snapshots
+    // that are NOT the valid best one.
+    // Transient memory usage is zero.
+    TransientResourceUsage result;
+    for (const auto& elem : _flushables) {
+        auto usage = elem.second.getFlusher()->get_transient_resource_usage();
+        result.merge(usage);
+    }
+    return result;
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
index b74e7e72a0e..65729767dbb 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
@@ -186,6 +186,8 @@ public:
     const ImportedAttributesRepo *getImportedAttributes() const override { return _importedAttributes.get(); }
 
     std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override;
+
+    TransientResourceUsage get_transient_resource_usage() const override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
index 1a0fdcb32aa..de35ab7394f 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
@@ -58,6 +58,8 @@ public:
     std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override;
 
     void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override;
+
+    TransientResourceUsage get_transient_resource_usage() const override { return {}; }
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
index f1b7eac3712..8101b29d98c 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
@@ -178,6 +178,12 @@ FlushableAttribute::FlushableAttribute(AttributeVectorSP attr,
 
 FlushableAttribute::~FlushableAttribute() = default;
 
+TransientResourceUsage
+FlushableAttribute::get_transient_resource_usage() const
+{
+    return _attrDir->get_transient_resource_usage();
+}
+
 IFlushTarget::SerialNum
 FlushableAttribute::getFlushedSerialNum() const
 {
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h
index 39d79372f25..e25422792fc 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h
@@ -2,10 +2,9 @@
 
 #pragma once
 
+#include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcorespi/flush/iflushtarget.h>
 #include <vespa/searchlib/common/tunefileinfo.h>
-#include <vespa/searchcore/proton/common/hw_info.h>
-
 
 namespace search { class AttributeVector; }
 
@@ -14,8 +13,8 @@ namespace vespalib { class ISequencedTaskExecutor; }
 
 namespace proton {
 
-
 class AttributeDirectory;
+class TransientResourceUsage;
 
 /**
  * Implementation of IFlushTarget interface for attribute vectors.
@@ -59,11 +58,12 @@ public:
                        vespalib::ISequencedTaskExecutor &attributeFieldWriter,
                        const HwInfo &hwInfo);
 
-    virtual
-    ~FlushableAttribute();
+    virtual ~FlushableAttribute();
 
     void setCleanUpAfterFlush(bool cleanUp) { _cleanUpAfterFlush = cleanUp; }
 
+    TransientResourceUsage get_transient_resource_usage() const;
+
     // Implements IFlushTarget
     virtual MemoryGain getApproxMemoryGain() const override;
     virtual DiskGain getApproxDiskGain() const override;
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
index 4b6b8dc687c..ce163827d42 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
@@ -6,6 +6,7 @@
 #include "exclusive_attribute_read_accessor.h"
 #include "i_attribute_factory.h"
 #include <vespa/searchcommon/attribute/i_attribute_functor.h>
+#include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h>
 #include <vespa/searchcorespi/flush/iflushtarget.h>
 #include <vespa/searchlib/attribute/iattributemanager.h>
 #include <vespa/searchlib/common/serialnum.h>
@@ -105,6 +106,8 @@ struct IAttributeManager : public search::IAttributeManager
     virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) = 0;
 
     virtual const ImportedAttributesRepo *getImportedAttributes() const = 0;
+
+    virtual TransientResourceUsage get_transient_resource_usage() const = 0;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
index 609ee585a6c..a712035e9af 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
@@ -166,9 +166,13 @@ DocumentMetaStoreFlushTarget(const DocumentMetaStore::SP dms, ITlsSyncer &tlsSyn
     _lastStats.setPathElementsToLog(8);
 }
 
-
 DocumentMetaStoreFlushTarget::~DocumentMetaStoreFlushTarget() = default;
 
+TransientResourceUsage
+DocumentMetaStoreFlushTarget::get_transient_resource_usage() const
+{
+    return _dmsDir->get_transient_resource_usage();
+}
 
 IFlushTarget::SerialNum
 DocumentMetaStoreFlushTarget::getFlushedSerialNum() const
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h
index 17072d28515..ef9f9299791 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h
@@ -10,10 +10,11 @@ namespace search::common { class FileHeaderContext; }
 
 namespace proton {
 
-class ITlsSyncer;
-class AttributeDiskLayout;
 class AttributeDirectory;
+class AttributeDiskLayout;
 class DocumentMetaStore;
+class ITlsSyncer;
+class TransientResourceUsage;
 
 /**
  * Implementation of IFlushTarget interface for document meta store.
@@ -54,6 +55,8 @@ public:
 
     void setCleanUpAfterFlush(bool cleanUp) { _cleanUpAfterFlush = cleanUp; }
 
+    TransientResourceUsage get_transient_resource_usage() const;
+
     MemoryGain getApproxMemoryGain() const override;
     DiskGain getApproxDiskGain() const override;
     Time getLastFlushTime() const override;
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 6cd1a4ec728..e4d432bf4fd 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -110,14 +110,12 @@ public:
     explicit DocumentDBResourceUsageProvider(const DocumentDB& doc_db) noexcept
         : _doc_db(doc_db)
     {}
+
     TransientResourceUsage get_transient_resource_usage() const override {
-        // Transient disk usage is measured as the total disk usage of all current fusion indexes.
-        // Transient memory usage is measured as the total memory usage of all memory indexes.
         if (!_doc_db.get_state().get_load_done()) {
             return {0, 0};
         }
-        auto stats = _doc_db.getReadySubDB()->getSearchableStats();
-        return {stats.fusion_size_on_disk(), stats.memoryUsage().allocatedBytes()};
+        return _doc_db.getReadySubDB()->get_transient_resource_usage();
     }
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
index ebe20f24d92..d5475d1f904 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
@@ -352,4 +352,12 @@ FastAccessDocSubDB::getNewestFlushedSerial()
     return highest;
 }
 
+TransientResourceUsage
+FastAccessDocSubDB::get_transient_resource_usage() const
+{
+    auto result = StoreOnlyDocSubDB::get_transient_resource_usage();
+    result.merge(getAttributeManager()->get_transient_resource_usage());
+    return result;
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
index d29c71ea43c..94fca94c75d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
@@ -121,6 +121,7 @@ public:
     SerialNum getOldestFlushedSerial() override;
     SerialNum getNewestFlushedSerial() override;
     virtual void pruneRemovedFields(SerialNum serialNum) override;
+    TransientResourceUsage get_transient_resource_usage() const override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
index f84352a4558..b945c67660b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
@@ -28,11 +28,7 @@ class DocumentDBConfig;
 class DocumentSubDbInitializer;
 class DocumentSubDbInitializerResult;
 class FeedHandler;
-struct IAttributeManager;
-struct IBucketStateCalculator;
-struct IDocumentDBReferenceResolver;
 class IDocumentDBReference;
-struct IDocumentMetaStoreContext;
 class IDocumentRetriever;
 class IFeedView;
 class IIndexWriter;
@@ -40,9 +36,14 @@ class IReplayConfig;
 class ISearchHandler;
 class ISummaryAdapter;
 class ISummaryManager;
+class PendingLidTrackerBase;
 class ReconfigParams;
 class RemoveDocumentsOperation;
-class PendingLidTrackerBase;
+class TransientResourceUsage;
+struct IAttributeManager;
+struct IBucketStateCalculator;
+struct IDocumentDBReferenceResolver;
+struct IDocumentMetaStoreContext;
 
 /**
  * Interface for a document sub database that handles a subset of the documents that belong to a
@@ -123,6 +124,7 @@ public:
     virtual void tearDownReferences(IDocumentDBReferenceResolver &resolver) = 0;
     virtual void validateDocStore(FeedHandler &op, SerialNum serialNum) const = 0;
     virtual PendingLidTrackerBase & getUncommittedLidsTracker() = 0;
+    virtual TransientResourceUsage get_transient_resource_usage() const = 0;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
index b623b461f6e..043e9cd5d3f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
@@ -343,4 +343,15 @@ SearchableDocSubDB::clearViews() {
     Parent::clearViews();
 }
 
+TransientResourceUsage
+SearchableDocSubDB::get_transient_resource_usage() const
+{
+    auto result = FastAccessDocSubDB::get_transient_resource_usage();
+    // Transient disk usage is measured as the total disk usage of all current fusion indexes.
+    // Transient memory usage is measured as the total memory usage of all memory indexes.
+    auto stats = getSearchableStats();
+    result.merge({stats.fusion_size_on_disk(), stats.memoryUsage().allocatedBytes()});
+    return result;
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
index d264a625e96..c628d9a96b7 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
@@ -131,6 +131,7 @@ public:
     void close() override;
     std::shared_ptr<IDocumentDBReference> getDocumentDBReference() override;
     void tearDownReferences(IDocumentDBReferenceResolver &resolver) override;
+    TransientResourceUsage get_transient_resource_usage() const override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
index 032307c1157..9419dfa1c90 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
@@ -585,4 +585,10 @@ addTags(vespalib::GenericHeader &header, const vespalib::string &name) const
     header.putTag(Tag("subDB", _subDB));
 }
 
+TransientResourceUsage
+StoreOnlyDocSubDB::get_transient_resource_usage() const
+{
+    return _dmsFlushTarget->get_transient_resource_usage();
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
index cb1f1ed07bb..f694cc7298f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
@@ -236,6 +236,7 @@ public:
     PendingLidTrackerBase & getUncommittedLidsTracker() override { return *_pendingLidsForCommit; }
     vespalib::datastore::CompactionStrategy computeCompactionStrategy(vespalib::datastore::CompactionStrategy strategy) const;
     bool isNodeRetired() const { return _nodeRetired; }
+    TransientResourceUsage get_transient_resource_usage() const override;
 
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
index 6632fbc856a..6c142c97aaf 100644
--- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
@@ -103,6 +103,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB
     }
 
     void tearDownReferences(IDocumentDBReferenceResolver &) override { }
+    TransientResourceUsage get_transient_resource_usage() const override { return {}; }
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h
index 75bb3291dd0..987d60dff01 100644
--- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h
@@ -105,6 +105,7 @@ public:
     std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override {
         return _mock.readable_attribute_vector(name);
     }
+    TransientResourceUsage get_transient_resource_usage() const override { return {}; }
 };
 
 }