Continue serving search queries when in Maintenance node state

Previously, entering maintenance state would implicitly deactivate
all buckets on the searchnode and cause empty responses to be returned
for searches.

However, container query dispatch uses async health pings to decide
which nodes to route queries to, so it would be possible for a node to
still be used for queries for a few seconds until the ping discovered
that the node should not be used. In the case of multiple groups without
multiple ready replicas within the group, this would cause transient
coverage loss since the dispatcher would not realize it should route
queries to other groups instead.

With this commit, maintenance edge behavior is changed as follows:

 - Buckets are _not_ deactivated when going from an available state
   to the maintenance state. However, they _are_ deactivate when going
   from maintenance state to an available state in order to avoid transient
   query duplicates immediately after the change.
 - Searches are executed as normal instead of returning empty replies
   when the node is in maintenance state.

The following behavior is intentionally _not_ changed:

 - The search interface is still marked as offline when in maintenance
   state, as this signals that the node should be taken out of rotation.
   In particular, it's critical that the RPC health ping response is
   explicitly tagged as having zero active docs when the search interface
   is offline, even though many buckets may now actually be active.
   Otherwise, queries would not be gracefully drained from the node.

3 files changed, 46 insertions, 10 deletions

diff --git a/persistence/src/tests/spi/clusterstatetest.cpp b/persistence/src/tests/spi/clusterstatetest.cpp
index 8a303c1a1ac..9f7507c992b 100644
--- a/persistence/src/tests/spi/clusterstatetest.cpp
+++ b/persistence/src/tests/spi/clusterstatetest.cpp
@@ -233,4 +233,31 @@ TEST(ClusterStateTest, can_infer_own_node_retired_state)
     EXPECT_TRUE(!node_marked_as_retired_in_state("distributor:3 storage:3 .1.s:r", d, 0));
 }
 
+namespace {
+
+bool
+node_marked_as_maintenance_in_state(const std::string& stateStr,
+                                    const lib::Distribution& d,
+                                    uint16_t node)
+{
+    lib::ClusterState s(stateStr);
+    ClusterState state(s, node, d);
+    return state.nodeMaintenance();
+}
+
+}
+
+TEST(ClusterStateTest, can_infer_own_node_maintenance_state)
+{
+    lib::Distribution d(lib::Distribution::getDefaultDistributionConfig(3, 3));
+
+    EXPECT_FALSE(node_marked_as_maintenance_in_state("distributor:3 storage:3", d, 0));
+    EXPECT_FALSE(node_marked_as_maintenance_in_state("distributor:3 storage:3 .0.s:i", d, 0));
+    EXPECT_FALSE(node_marked_as_maintenance_in_state("distributor:3 storage:3 .0.s:d", d, 0));
+    EXPECT_TRUE( node_marked_as_maintenance_in_state("distributor:3 storage:3 .0.s:m", d, 0));
+    EXPECT_FALSE(node_marked_as_maintenance_in_state("distributor:3 storage:3 .0.s:r", d, 0));
+    EXPECT_FALSE(node_marked_as_maintenance_in_state("distributor:3 storage:3 .0.s:m", d, 1));
+    EXPECT_FALSE(node_marked_as_maintenance_in_state("distributor:3 storage:3 .1.s:m", d, 0));
+}
+
 }
diff --git a/persistence/src/vespa/persistence/spi/clusterstate.cpp b/persistence/src/vespa/persistence/spi/clusterstate.cpp
index 4bc538996ca..a244b607391 100644
--- a/persistence/src/vespa/persistence/spi/clusterstate.cpp
+++ b/persistence/src/vespa/persistence/spi/clusterstate.cpp
@@ -68,28 +68,32 @@ ClusterState::shouldBeReady(const Bucket& b) const {
     return Trinary::False;
 }
 
-bool ClusterState::clusterUp() const {
+bool ClusterState::clusterUp() const noexcept {
     return _state && _state->getClusterState() == lib::State::UP;
 }
 
-bool ClusterState::nodeHasStateOneOf(const char* states) const {
+bool ClusterState::nodeHasStateOneOf(const char* states) const noexcept {
     return _state &&
            _state->getNodeState(lib::Node(lib::NodeType::STORAGE, _nodeIndex)).
                    getState().oneOf(states);
 }
 
-bool ClusterState::nodeUp() const {
+bool ClusterState::nodeUp() const noexcept {
     return nodeHasStateOneOf("uir");
 }
 
-bool ClusterState::nodeInitializing() const {
+bool ClusterState::nodeInitializing() const noexcept {
     return nodeHasStateOneOf("i");
 }
 
-bool ClusterState::nodeRetired() const {
+bool ClusterState::nodeRetired() const noexcept {
     return nodeHasStateOneOf("r");
 }
 
+bool ClusterState::nodeMaintenance() const noexcept {
+    return nodeHasStateOneOf("m");
+}
+
 void ClusterState::serialize(vespalib::nbostream& o) const {
     assert(_distribution);
     assert(_state);
diff --git a/persistence/src/vespa/persistence/spi/clusterstate.h b/persistence/src/vespa/persistence/spi/clusterstate.h
index 8e48758e243..df556d09cb7 100644
--- a/persistence/src/vespa/persistence/spi/clusterstate.h
+++ b/persistence/src/vespa/persistence/spi/clusterstate.h
@@ -45,23 +45,28 @@ public:
      * compared to the complete list of nodes, and deigns the system to be
      * unusable.
      */
-    bool clusterUp() const;
+    bool clusterUp() const noexcept;
 
     /**
      * Returns false if this node has been set in a state where it should not
      * receive external load.
      */
-    bool nodeUp() const;
+    bool nodeUp() const noexcept;
 
     /**
      * Returns true iff this node is marked as Initializing in the cluster state.
      */
-    bool nodeInitializing() const;
+    bool nodeInitializing() const noexcept;
 
     /**
      * Returns true iff this node is marked as Retired in the cluster state.
      */
-    bool nodeRetired() const;
+    bool nodeRetired() const noexcept;
+
+    /**
+     * Returns true iff this node is marked as Maintenance in the cluster state.
+     */
+    bool nodeMaintenance() const noexcept;
 
     /**
      * Returns a serialized form of this object.
@@ -74,7 +79,7 @@ private:
     uint16_t _nodeIndex;
 
     void deserialize(vespalib::nbostream&);
-    bool nodeHasStateOneOf(const char* states) const;
+    bool nodeHasStateOneOf(const char* states) const noexcept;
 };
 
 }