Sample and track the current transient memory usage across all document dbs.

Previously, the transient memory usage was the theoretical worst case needed during loading of attributes.
Now, we change this to being a snapshot of the total memory used by all memory indexes.
In a simular fashion we plan to track the transient disk usage of all ongoing fusion disk indexes.

Eventually, the transient resource usage will be subtracted from the absolute resource usage,
before this is reported to the cluster controller in order to decide whether to block feeding.
By doing this the transient resource usage is covered by the headroom on the content node,
instead of leading to feed blocked due to natural fluctuations.

3 files changed, 21 insertions, 65 deletions

diff --git a/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/CMakeLists.txt
new file mode 100644
index 00000000000..bc77f2d949b
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_transient_memory_calculator_test_app TEST
+    SOURCES
+    attribute_transient_memory_calculator_test.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_pcommon
+    GTest::GTest
+)
+vespa_add_test(NAME searchcore_attribute_transient_memory_calculator_test_app COMMAND searchcore_attribute_transient_memory_calculator_test_app)
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/attribute_transient_memory_calculator_test.cpp
index 31f90075421..786d1f6a801 100644
--- a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/attribute_transient_memory_calculator_test.cpp
@@ -5,10 +5,7 @@
 #include <vespa/searchlib/attribute/attributevector.h>
 #include <vespa/searchlib/attribute/integerbase.h>
 #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h>
-#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h>
-#include <vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h>
-#include <vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h>
-#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h>
+#include <vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.h>
 #include <vespa/vespalib/gtest/gtest.h>
 #include <vespa/vespalib/stllike/string.h>
 
@@ -58,73 +55,32 @@ std::shared_ptr<AttributeVector> build_attribute_vector(const vespalib::string&
 
 }
 
-class AttributeUsageSamplerFunctorTest : public ::testing::Test {
-protected:
-    AttributeUsageFilter                          _filter;
-    std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider;
-public:
-    AttributeUsageSamplerFunctorTest();
-    ~AttributeUsageSamplerFunctorTest();
-protected:
-    void sample_usage(bool sample_a1, bool sample_a2, bool old_fast_search, bool new_fast_search = true);
-    size_t get_transient_memory_usage() const { return _transient_usage_provider->get_transient_memory_usage(); }
-};
-
-AttributeUsageSamplerFunctorTest::AttributeUsageSamplerFunctorTest()
-    : _filter(),
-      _transient_usage_provider(std::make_shared<TransientResourceUsageProvider>())
-{
-}
-
-AttributeUsageSamplerFunctorTest::~AttributeUsageSamplerFunctorTest() = default;
-
-void
-AttributeUsageSamplerFunctorTest::sample_usage(bool sample_a1, bool sample_a2, bool old_fast_search, bool new_fast_search)
+size_t
+sample_usage(bool old_fast_search, bool new_fast_search)
 {
     auto old_config = build_config(old_fast_search);
     auto old_inspector = std::make_shared<AttributeConfigInspector>(old_config);
     auto av1 = build_attribute_vector("a1", *old_inspector, 1);
-    auto av2 = build_attribute_vector("a2", *old_inspector, 3);
     EXPECT_EQ(av1->getEnumeratedSave(), old_fast_search);
     auto new_config = build_config(new_fast_search);
     auto new_inspector = std::make_shared<AttributeConfigInspector>(new_config);
-    auto context = std::make_shared<AttributeUsageSamplerContext>(_filter, new_inspector, _transient_usage_provider);
-    if (sample_a1) {
-        AttributeUsageSamplerFunctor functor1(context, "ready");
-        functor1(*av1);
-    }
-    if (sample_a2) {
-        AttributeUsageSamplerFunctor functor2(context, "ready");
-        functor2(*av2);
-    }
-}
-
-TEST_F(AttributeUsageSamplerFunctorTest, plain_attribute_vector_requires_no_transient_memory_for_load)
-{
-    sample_usage(true, true, false, false);
-    EXPECT_EQ(0u, get_transient_memory_usage());
+    AttributeTransientMemoryCalculator calc;
+    return calc(*av1, *new_inspector->get_config("a1"));
 }
 
-TEST_F(AttributeUsageSamplerFunctorTest, fast_search_attribute_vector_requires_transient_memory_for_load)
+TEST(AttributeTransientMemoryCalculator, plain_attribute_vector_requires_no_transient_memory_for_load)
 {
-    sample_usage(true, false, true, true);
-    EXPECT_EQ(24u, get_transient_memory_usage());
+    EXPECT_EQ(0, sample_usage(false, false));
 }
 
-TEST_F(AttributeUsageSamplerFunctorTest, fast_search_attribute_vector_requires_more_transient_memory_for_load_from_unenumerated)
+TEST(AttributeTransientMemoryCalculator, fast_search_attribute_vector_requires_transient_memory_for_load)
 {
-    sample_usage(true, false, false, true);
-    EXPECT_EQ(40u, get_transient_memory_usage());
+    EXPECT_EQ(24u, sample_usage(true, true));
 }
 
-TEST_F(AttributeUsageSamplerFunctorTest, transient_memory_aggregation_function_for_attribute_usage_sampler_context_is_max)
+TEST(AttributeTransientMemoryCalculator, fast_search_attribute_vector_requires_more_transient_memory_for_load_from_unenumerated)
 {
-    sample_usage(true, false, true, true);
-    EXPECT_EQ(24u, get_transient_memory_usage());
-    sample_usage(false, true, true, true);
-    EXPECT_EQ(72u, get_transient_memory_usage());
-    sample_usage(true, true, true, true);
-    EXPECT_EQ(72u, get_transient_memory_usage());
+    EXPECT_EQ(40u, sample_usage(false, true));
 }
 
 }
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt
deleted file mode 100644
index 6034f5a1309..00000000000
--- a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchcore_attribute_usage_sampler_functor_test_app TEST
-    SOURCES
-    attribute_usage_sampler_functor_test.cpp
-    DEPENDS
-    searchcore_attribute
-    searchcore_pcommon
-    GTest::GTest
-)
-vespa_add_test(NAME searchcore_attribute_usage_sampler_functor_test_app COMMAND searchcore_attribute_usage_sampler_functor_test_app)