Merge pull request #6513 from vespa-engine/toregge/deactivate-content-layer-buckets-when-derived-cluster-state-indicates-node-down-edge

Deactivate all buckets in content layer bucket db when derived cluster

4 files changed, 30 insertions, 12 deletions

diff --git a/storage/src/vespa/storage/common/content_bucket_space.cpp b/storage/src/vespa/storage/common/content_bucket_space.cpp
index 2bae118e364..0827c721100 100644
--- a/storage/src/vespa/storage/common/content_bucket_space.cpp
+++ b/storage/src/vespa/storage/common/content_bucket_space.cpp
@@ -9,7 +9,8 @@ ContentBucketSpace::ContentBucketSpace(document::BucketSpace bucketSpace)
       _bucketDatabase(),
       _lock(),
       _clusterState(),
-      _distribution()
+      _distribution(),
+      _nodeUpInLastNodeStateSeenByProvider(false)
 {
 }
 
@@ -41,4 +42,18 @@ ContentBucketSpace::getDistribution() const
     return _distribution;
 }
 
+bool
+ContentBucketSpace::getNodeUpInLastNodeStateSeenByProvider() const
+{
+    std::lock_guard guard(_lock);
+    return _nodeUpInLastNodeStateSeenByProvider;
+}
+
+void
+ContentBucketSpace::setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider)
+{
+    std::lock_guard guard(_lock);
+    _nodeUpInLastNodeStateSeenByProvider = nodeUpInLastNodeStateSeenByProvider;
+}
+
 }
diff --git a/storage/src/vespa/storage/common/content_bucket_space.h b/storage/src/vespa/storage/common/content_bucket_space.h
index 6ccc82bc4fb..81ce6234879 100644
--- a/storage/src/vespa/storage/common/content_bucket_space.h
+++ b/storage/src/vespa/storage/common/content_bucket_space.h
@@ -22,6 +22,7 @@ private:
     mutable std::mutex _lock;
     std::shared_ptr<const lib::ClusterState> _clusterState;
     std::shared_ptr<const lib::Distribution> _distribution;
+    bool _nodeUpInLastNodeStateSeenByProvider;
 
 public:
     using UP = std::unique_ptr<ContentBucketSpace>;
@@ -33,6 +34,8 @@ public:
     std::shared_ptr<const lib::ClusterState> getClusterState() const;
     void setDistribution(std::shared_ptr<const lib::Distribution> distribution);
     std::shared_ptr<const lib::Distribution> getDistribution() const;
+    bool getNodeUpInLastNodeStateSeenByProvider() const;
+    void setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider);
 };
 
 }
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
index 961af1532a1..d1f0a24178a 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
@@ -36,7 +36,6 @@ FileStorManager(const config::ConfigUri & configUri, const spi::PartitionStateLi
       _partitions(partitions),
       _providerCore(provider),
       _providerErrorWrapper(_providerCore),
-      _nodeUpInLastNodeStateSeenByProvider(false),
       _providerMetric(new spi::MetricPersistenceProvider(_providerErrorWrapper)),
       _provider(_providerMetric.get()),
       _bucketIdFactory(_component.getBucketIdFactory()),
@@ -886,21 +885,23 @@ FileStorManager::updateState()
     auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle();
     lib::ClusterState::CSP state(clusterStateBundle->getBaselineClusterState());
     lib::Node node(_component.getNodeType(), _component.getIndex());
-    bool nodeUp = state->getNodeState(node).getState().oneOf("uir");
 
     LOG(debug, "FileStorManager received cluster state '%s'", state->toString().c_str());
-    // If edge where we go down
-    if (_nodeUpInLastNodeStateSeenByProvider && !nodeUp) {
-        LOG(debug, "Received cluster state where this node is down; de-activating all buckets in database");
-        Deactivator deactivator;
-        _component.getBucketSpaceRepo().forEachBucket(deactivator, "FileStorManager::updateState");
-    }
     for (const auto &elem : _component.getBucketSpaceRepo()) {
         BucketSpace bucketSpace(elem.first);
-        spi::ClusterState spiState(*elem.second->getClusterState(), _component.getIndex(), *elem.second->getDistribution());
+        ContentBucketSpace &contentBucketSpace = *elem.second;
+        auto derivedClusterState = contentBucketSpace.getClusterState();
+        bool nodeUp = derivedClusterState->getNodeState(node).getState().oneOf("uir");
+        // If edge where we go down
+        if (contentBucketSpace.getNodeUpInLastNodeStateSeenByProvider() && !nodeUp) {
+            LOG(debug, "Received cluster state where this node is down; de-activating all buckets in database for bucket space %s", bucketSpace.toString().c_str());
+            Deactivator deactivator;
+            contentBucketSpace.bucketDatabase().all(deactivator, "FileStorManager::updateState");
+        }
+        contentBucketSpace.setNodeUpInLastNodeStateSeenByProvider(nodeUp);
+        spi::ClusterState spiState(*derivedClusterState, _component.getIndex(), *contentBucketSpace.getDistribution());
         _provider->setClusterState(bucketSpace, spiState);
     }
-    _nodeUpInLastNodeStateSeenByProvider = nodeUp;
 }
 
 void
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
index 0bd2ffa5910..4bf2c1049cf 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
@@ -56,7 +56,6 @@ class FileStorManager : public StorageLinkQueued,
     const spi::PartitionStateList& _partitions;
     spi::PersistenceProvider& _providerCore;
     ProviderErrorWrapper _providerErrorWrapper;
-    bool _nodeUpInLastNodeStateSeenByProvider;
     spi::MetricPersistenceProvider::UP _providerMetric;
     spi::PersistenceProvider* _provider;