Merge pull request #29269 from vespa-engine/geirst/control-resource-usage-when-in-maintenance

Control resource usage when node in maintenance

5 files changed, 75 insertions, 47 deletions

diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt
index 0b42d480e33..b89d4d9301f 100644
--- a/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt
+++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt
@@ -10,5 +10,6 @@ vespa_add_executable(searchcore_clusterstatehandler_test_app TEST
     searchcore_attribute
     searchcore_pcommon
     searchcore_grouping
+    GTest::GTest
 )
 vespa_add_test(NAME searchcore_clusterstatehandler_test_app COMMAND searchcore_clusterstatehandler_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp b/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp
index 4cd2eab513c..539ee9ce3aa 100644
--- a/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp
@@ -1,11 +1,12 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include <vespa/searchcore/proton/server/clusterstatehandler.h>
 #include <vespa/searchcore/proton/server/iclusterstatechangedhandler.h>
 #include <vespa/searchcore/proton/test/test.h>
-#include <vespa/vespalib/util/threadstackexecutor.h>
-#include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vdslib/distribution/distribution.h>
 #include <vespa/vdslib/state/clusterstate.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("cluster_state_handler_test");
@@ -17,8 +18,7 @@ using storage::spi::BucketIdListResult;
 using storage::spi::ClusterState;
 using storage::spi::Result;
 
-struct MyClusterStateChangedHandler : public IClusterStateChangedHandler
-{
+struct MyClusterStateChangedHandler : public IClusterStateChangedHandler {
     std::shared_ptr<IBucketStateCalculator> _calc;
     void
     notifyClusterStateChanged(const std::shared_ptr<IBucketStateCalculator> &newCalc) override {
@@ -26,23 +26,26 @@ struct MyClusterStateChangedHandler : public IClusterStateChangedHandler
     }
 };
 
-
 BucketId bucket1(1);
 BucketId bucket2(2);
 BucketId bucket3(3);
 Distribution distribution(Distribution::getDefaultDistributionConfig(3, 3));
-storage::lib::ClusterState rawClusterState("version:1 storage:3 distributor:3");
-ClusterState clusterState(rawClusterState, 0, distribution);
 
+ClusterState make_cluster_state(const vespalib::string& state, uint16_t node_index, bool maintenance_in_all_spaces = false) {
+    return ClusterState(storage::lib::ClusterState(state), node_index, distribution, maintenance_in_all_spaces);
+}
 
-struct Fixture
-{
+ClusterState basic_state = make_cluster_state("distributor:3 storage:3", 0);
+ClusterState node_retired_state = make_cluster_state("distributor:3 .1.s:d storage:3 .1.s:r", 1);
+ClusterState node_maintenance_state = make_cluster_state("distributor:3 storage:3", 1, true);
+
+struct ClusterStateHandlerTest : testing::Test {
     vespalib::ThreadStackExecutor   _exec;
     ClusterStateHandler             _stateHandler;
     MyClusterStateChangedHandler    _changedHandler;
     test::GenericResultHandler      _genericHandler;
     test::BucketIdListResultHandler _bucketListHandler;
-    Fixture()
+    ClusterStateHandlerTest()
         : _exec(1),
           _stateHandler(_exec),
           _changedHandler(),
@@ -51,47 +54,66 @@ struct Fixture
     {
         _stateHandler.addClusterStateChangedHandler(&_changedHandler);
     }
-    ~Fixture()
-    {
+    ~ClusterStateHandlerTest() {
         _stateHandler.removeClusterStateChangedHandler(&_changedHandler);
     }
+    const IBucketStateCalculator& set_cluster_state(const ClusterState& state) {
+        _stateHandler.handleSetClusterState(state, _genericHandler);
+        _exec.sync();
+        EXPECT_TRUE(_changedHandler._calc);
+        return *_changedHandler._calc;
+    }
 };
 
+TEST_F(ClusterStateHandlerTest, cluster_state_change_is_notified)
+{
+    const auto& calc = set_cluster_state(basic_state);
+    EXPECT_TRUE(calc.clusterUp());
+    EXPECT_TRUE(calc.nodeUp());
+    EXPECT_FALSE(calc.nodeInitializing());
+    EXPECT_FALSE(calc.nodeRetired());
+    EXPECT_FALSE(calc.nodeMaintenance());
+    EXPECT_FALSE(calc.node_retired_or_maintenance());
+}
 
-TEST_F("require that cluster state change is notified", Fixture)
+TEST_F(ClusterStateHandlerTest, node_in_retired_state)
 {
-    f._stateHandler.handleSetClusterState(clusterState, f._genericHandler);
-    f._exec.sync();
-    EXPECT_TRUE(f._changedHandler._calc);
+    const auto &calc = set_cluster_state(node_retired_state);
+    EXPECT_TRUE(calc.nodeRetired());
+    EXPECT_FALSE(calc.nodeMaintenance());
+    EXPECT_TRUE(calc.node_retired_or_maintenance());
 }
 
+TEST_F(ClusterStateHandlerTest, node_in_maintenance_state)
+{
+    const auto &calc = set_cluster_state(node_maintenance_state);
+    EXPECT_FALSE(calc.nodeRetired());
+    EXPECT_TRUE(calc.nodeMaintenance());
+    EXPECT_TRUE(calc.node_retired_or_maintenance());
+}
 
-TEST_F("require that modified buckets are returned", Fixture)
+TEST_F(ClusterStateHandlerTest, modified_buckets_are_returned)
 {
-    f._stateHandler.handleSetClusterState(clusterState, f._genericHandler);
-    f._exec.sync();
+    _stateHandler.handleSetClusterState(basic_state, _genericHandler);
+    _exec.sync();
 
     // notify 2 buckets
-    IBucketModifiedHandler &bmh = f._stateHandler;
+    IBucketModifiedHandler &bmh = _stateHandler;
     bmh.notifyBucketModified(bucket1);
     bmh.notifyBucketModified(bucket2);
-    f._stateHandler.handleGetModifiedBuckets(f._bucketListHandler);
-    f._exec.sync();
-    EXPECT_EQUAL(2u, f._bucketListHandler.getList().size());
-    EXPECT_EQUAL(bucket1, f._bucketListHandler.getList()[0]);
-    EXPECT_EQUAL(bucket2, f._bucketListHandler.getList()[1]);
+    _stateHandler.handleGetModifiedBuckets(_bucketListHandler);
+    _exec.sync();
+    EXPECT_EQ(2u, _bucketListHandler.getList().size());
+    EXPECT_EQ(bucket1, _bucketListHandler.getList()[0]);
+    EXPECT_EQ(bucket2, _bucketListHandler.getList()[1]);
 
     // notify 1 bucket, already reported buckets should be gone
     bmh.notifyBucketModified(bucket3);
-    f._stateHandler.handleGetModifiedBuckets(f._bucketListHandler);
-    f._exec.sync();
-    EXPECT_EQUAL(1u, f._bucketListHandler.getList().size());
-    EXPECT_EQUAL(bucket3, f._bucketListHandler.getList()[0]);
+    _stateHandler.handleGetModifiedBuckets(_bucketListHandler);
+    _exec.sync();
+    EXPECT_EQ(1u, _bucketListHandler.getList().size());
+    EXPECT_EQ(bucket3, _bucketListHandler.getList()[0]);
 }
 
-
-TEST_MAIN()
-{
-    TEST_RUN_ALL();
-}
+GTEST_MAIN_RUN_ALL_TESTS()
 
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index b5096a4c046..596ebb933a4 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -620,30 +620,35 @@ TEST_F("require that attribute manager can be reconfigured", SearchableFixture)
     requireThatAttributeManagerCanBeReconfigured(f);
 }
 
-TEST_F("require that subdb reflect retirement", FastAccessFixture)
+TEST_F("require that subdb reflect retirement or maintenance", FastAccessFixture)
 {
     CompactionStrategy cfg(0.1, 0.3);
 
-    EXPECT_FALSE(f._subDb.isNodeRetired());
+    EXPECT_FALSE(f._subDb.is_node_retired_or_maintenance());
     auto unretired_cfg = f._subDb.computeCompactionStrategy(cfg);
     EXPECT_TRUE(cfg == unretired_cfg);
 
     auto calc = std::make_shared<proton::test::BucketStateCalculator>();
     calc->setNodeRetired(true);
     f.setBucketStateCalculator(calc);
-    EXPECT_TRUE(f._subDb.isNodeRetired());
+    EXPECT_TRUE(f._subDb.is_node_retired_or_maintenance());
     auto retired_cfg = f._subDb.computeCompactionStrategy(cfg);
     EXPECT_TRUE(cfg != retired_cfg);
     EXPECT_TRUE(CompactionStrategy(0.5, 0.5) == retired_cfg);
 
     calc->setNodeRetired(false);
+    calc->setNodeMaintenance(true);
     f.setBucketStateCalculator(calc);
-    EXPECT_FALSE(f._subDb.isNodeRetired());
+    EXPECT_TRUE(f._subDb.is_node_retired_or_maintenance());
+
+    calc->setNodeMaintenance(false);
+    f.setBucketStateCalculator(calc);
+    EXPECT_FALSE(f._subDb.is_node_retired_or_maintenance());
     unretired_cfg = f._subDb.computeCompactionStrategy(cfg);
     EXPECT_TRUE(cfg == unretired_cfg);
 }
 
-TEST_F("require that attribute compaction config reflect retirement", FastAccessFixture) {
+TEST_F("require that attribute compaction config reflect retirement or maintenance", FastAccessFixture) {
     CompactionStrategy default_cfg(0.05, 0.2);
     CompactionStrategy retired_cfg(0.5, 0.5);
 
diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp
index d4d2a6dc377..eea523870c3 100644
--- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp
@@ -44,7 +44,7 @@ JobTestBase::init(uint32_t allowedLidBloat,
           double allowedLidBloatFactor,
           double resourceLimitFactor,
           vespalib::duration interval,
-          bool nodeRetired,
+          bool node_retired_or_maintenance,
           uint32_t maxOutstandingMoveOps)
 {
     _handler = std::make_shared<MyHandler>(maxOutstandingMoveOps != MAX_OUTSTANDING_MOVE_OPS, true);
@@ -57,7 +57,7 @@ JobTestBase::init(uint32_t allowedLidBloat,
     _master = std::make_unique<proton::SyncableExecutorThreadService> (*_singleExecutor);
     _bucketExecutor = std::make_unique<storage::spi::dummy::DummyBucketExecutor>(4);
     _job = lidspace::CompactionJob::create(compactCfg, RetainGuard(_refCount), _handler, _storer, *_master, *_bucketExecutor,
-                                           _diskMemUsageNotifier, blockableCfg, _clusterStateHandler, nodeRetired,
+                                           _diskMemUsageNotifier, blockableCfg, _clusterStateHandler, node_retired_or_maintenance,
                                            document::BucketSpace::placeHolder());
 }
 
@@ -180,10 +180,10 @@ JobTest::init(uint32_t allowedLidBloat,
               double allowedLidBloatFactor,
               double resourceLimitFactor,
               vespalib::duration interval,
-              bool nodeRetired,
+              bool node_retired_or_maintenance,
               uint32_t maxOutstandingMoveOps)
 {
-    JobTestBase::init(allowedLidBloat, allowedLidBloatFactor, resourceLimitFactor, interval, nodeRetired, maxOutstandingMoveOps);
+    JobTestBase::init(allowedLidBloat, allowedLidBloatFactor, resourceLimitFactor, interval, node_retired_or_maintenance, maxOutstandingMoveOps);
     _jobRunner = std::make_unique<MyDirectJobRunner>(*_job);
 }
 
diff --git a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp
index ec135800fde..79b4b4a3627 100644
--- a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp
+++ b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp
@@ -69,8 +69,8 @@ struct Fixture
     void notifyDiskMemUsage(const ResourceUsageState &diskState, const ResourceUsageState &memoryState) {
         updater.notifyDiskMemUsage(DiskMemUsageState(diskState, memoryState));
     }
-    void setNodeRetired(bool nodeRetired) {
-        updater.setNodeRetired(nodeRetired);
+    void set_node_retired_or_maintenance(bool value) {
+        updater.set_node_retired_or_maintenance(value);
     }
 };
 
@@ -226,12 +226,12 @@ TEST_F("require that we must go below low watermark for memory usage before usin
     TEST_DO(f.assertStrategyConfig(4, 1, 20));
 }
 
-TEST_F("require that more disk bloat is allowed while node state is retired", Fixture)
+TEST_F("require that more disk bloat is allowed while node state is retired or maintenance", Fixture)
 {
     constexpr double DEFAULT_DISK_BLOAT = 0.25;
     f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.3), belowLimit());
     TEST_DO(f.assertStrategyDiskConfig(DEFAULT_DISK_BLOAT, DEFAULT_DISK_BLOAT));
-    f.setNodeRetired(true);
+    f.set_node_retired_or_maintenance(true);
     TEST_DO(f.assertStrategyDiskConfig((0.8 - ((0.3/0.7)*(1 - DEFAULT_DISK_BLOAT))) / 0.8, 1.0));
     f.notifyDiskMemUsage(belowLimit(), belowLimit());
     TEST_DO(f.assertStrategyDiskConfig(DEFAULT_DISK_BLOAT, DEFAULT_DISK_BLOAT));