Merge pull request #17182 from vespa-engine/geirst/sample-transient-disk-usage-for-disk-index-fusion

Sample transient disk usage for disk index fusion

3 files changed, 54 insertions, 35 deletions

diff --git a/searchlib/src/tests/util/searchable_stats/CMakeLists.txt b/searchlib/src/tests/util/searchable_stats/CMakeLists.txt
index cbaabba41f3..786c316eb4f 100644
--- a/searchlib/src/tests/util/searchable_stats/CMakeLists.txt
+++ b/searchlib/src/tests/util/searchable_stats/CMakeLists.txt
@@ -4,5 +4,6 @@ vespa_add_executable(searchlib_searchable_stats_test_app TEST
     searchable_stats_test.cpp
     DEPENDS
     searchlib
+    GTest::GTest
 )
 vespa_add_test(NAME searchlib_searchable_stats_test_app COMMAND searchlib_searchable_stats_test_app)
diff --git a/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp b/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp
index c91214f5cfe..7f53eaddea5 100644
--- a/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp
+++ b/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp
@@ -1,41 +1,49 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/searchlib/util/searchable_stats.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
 #include <vespa/log/log.h>
 LOG_SETUP("searchable_stats_test");
-#include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/searchlib/util/searchable_stats.h>
 
 using namespace search;
 
-class Test : public vespalib::TestApp {
-public:
-    int Main() override;
-};
-
-int
-Test::Main()
+TEST(SearchableStatsTest, merge_also_tracks_max_size_on_disk_for_component)
 {
-    TEST_INIT("searchable_stats_test");
+    SearchableStats stats;
+    EXPECT_EQ(0u, stats.memoryUsage().allocatedBytes());
+    EXPECT_EQ(0u, stats.docsInMemory());
+    EXPECT_EQ(0u, stats.sizeOnDisk());
+    EXPECT_EQ(0u, stats.max_component_size_on_disk());
     {
-        SearchableStats stats;
-        EXPECT_EQUAL(0u, stats.memoryUsage().allocatedBytes());
-        EXPECT_EQUAL(0u, stats.docsInMemory());
-        EXPECT_EQUAL(0u, stats.sizeOnDisk());
-        {
-            SearchableStats rhs;
-            EXPECT_EQUAL(&rhs.memoryUsage(vespalib::MemoryUsage(100,0,0,0)), &rhs);
-            EXPECT_EQUAL(&rhs.docsInMemory(10), &rhs);
-            EXPECT_EQUAL(&rhs.sizeOnDisk(1000), &rhs);
-            EXPECT_EQUAL(&stats.add(rhs), &stats);
-        }
-        EXPECT_EQUAL(100u, stats.memoryUsage().allocatedBytes());
-        EXPECT_EQUAL(10u, stats.docsInMemory());
-        EXPECT_EQUAL(1000u, stats.sizeOnDisk());
-        EXPECT_EQUAL(&stats.add(SearchableStats().memoryUsage(vespalib::MemoryUsage(100,0,0,0)).docsInMemory(10).sizeOnDisk(1000)), &stats);
-        EXPECT_EQUAL(200u, stats.memoryUsage().allocatedBytes());
-        EXPECT_EQUAL(20u, stats.docsInMemory());
-        EXPECT_EQUAL(2000u, stats.sizeOnDisk());
+        SearchableStats rhs;
+        EXPECT_EQ(&rhs.memoryUsage(vespalib::MemoryUsage(100,0,0,0)), &rhs);
+        EXPECT_EQ(&rhs.docsInMemory(10), &rhs);
+        EXPECT_EQ(&rhs.sizeOnDisk(1000), &rhs);
+        EXPECT_EQ(1000u, rhs.max_component_size_on_disk());
+        EXPECT_EQ(&stats.merge(rhs), &stats);
     }
-    TEST_DONE();
+    EXPECT_EQ(100u, stats.memoryUsage().allocatedBytes());
+    EXPECT_EQ(10u, stats.docsInMemory());
+    EXPECT_EQ(1000u, stats.sizeOnDisk());
+    EXPECT_EQ(1000u, stats.max_component_size_on_disk());
+
+    stats.merge(SearchableStats()
+                        .memoryUsage(vespalib::MemoryUsage(150,0,0,0))
+                        .docsInMemory(15)
+                        .sizeOnDisk(1500));
+    EXPECT_EQ(250u, stats.memoryUsage().allocatedBytes());
+    EXPECT_EQ(25u, stats.docsInMemory());
+    EXPECT_EQ(2500u, stats.sizeOnDisk());
+    EXPECT_EQ(1500u, stats.max_component_size_on_disk());
+
+    stats.merge(SearchableStats()
+                        .memoryUsage(vespalib::MemoryUsage(120,0,0,0))
+                        .docsInMemory(12)
+                        .sizeOnDisk(1200));
+    EXPECT_EQ(370u, stats.memoryUsage().allocatedBytes());
+    EXPECT_EQ(37u, stats.docsInMemory());
+    EXPECT_EQ(3700u, stats.sizeOnDisk());
+    EXPECT_EQ(1500u, stats.max_component_size_on_disk());
 }
 
-TEST_APPHOOK(Test);
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/vespa/searchlib/util/searchable_stats.h b/searchlib/src/vespa/searchlib/util/searchable_stats.h
index 4772c04f6be..5138e636df5 100644
--- a/searchlib/src/vespa/searchlib/util/searchable_stats.h
+++ b/searchlib/src/vespa/searchlib/util/searchable_stats.h
@@ -6,9 +6,9 @@
 namespace search {
 
 /**
- * Simple statistics for a single Searchable component. Used for
- * internal aggregation before inserting numbers into the metrics
- * framework.
+ * Simple statistics for a single Searchable component or multiple components that are merged together.
+ *
+ * E.g. used for internal aggregation before inserting numbers into the metrics framework.
  **/
 class SearchableStats
 {
@@ -16,9 +16,10 @@ private:
     vespalib::MemoryUsage _memoryUsage;
     size_t _docsInMemory;
     size_t _sizeOnDisk;
+    size_t _max_component_size_on_disk;
 
 public:
-    SearchableStats() : _memoryUsage(), _docsInMemory(0), _sizeOnDisk(0) {}
+    SearchableStats() : _memoryUsage(), _docsInMemory(0), _sizeOnDisk(0), _max_component_size_on_disk(0) {}
     SearchableStats &memoryUsage(const vespalib::MemoryUsage &usage) {
         _memoryUsage = usage;
         return *this;
@@ -31,13 +32,22 @@ public:
     size_t docsInMemory() const { return _docsInMemory; }
     SearchableStats &sizeOnDisk(size_t value) {
         _sizeOnDisk = value;
+        _max_component_size_on_disk = value;
         return *this;
     }
     size_t sizeOnDisk() const { return _sizeOnDisk; }
-    SearchableStats &add(const SearchableStats &rhs) {
+
+    /**
+     * Returns the max disk size used by a single Searchable component,
+     * e.g. among the components that are merged into a SearchableStats instance via merge().
+     */
+    size_t max_component_size_on_disk() const { return _max_component_size_on_disk; }
+
+    SearchableStats &merge(const SearchableStats &rhs) {
         _memoryUsage.merge(rhs._memoryUsage);
         _docsInMemory += rhs._docsInMemory;
         _sizeOnDisk += rhs._sizeOnDisk;
+        _max_component_size_on_disk = std::max(_max_component_size_on_disk, rhs._sizeOnDisk);
         return *this;
     }
 };