Deactivate all buckets in content layer bucket db when derived cluster

state indicates that node changes from up to down state even if node
remains up according to baseline cluster state.

4 files changed, 30 insertions, 12 deletions

diff --git a/storage/src/vespa/storage/common/content_bucket_space.cpp b/storage/src/vespa/storage/common/content_bucket_space.cpp
index 2bae118e364..0827c721100 100644
--- a/storage/src/vespa/storage/common/content_bucket_space.cpp
+++ b/storage/src/vespa/storage/common/content_bucket_space.cpp
@@ -9,7 +9,8 @@ ContentBucketSpace::ContentBucketSpace(document::BucketSpace bucketSpace)
       _bucketDatabase(),
       _lock(),
       _clusterState(),
-      _distribution()
+      _distribution(),
+      _nodeUpInLastNodeStateSeenByProvider(false)
 {
 }
 
@@ -41,4 +42,18 @@ ContentBucketSpace::getDistribution() const
     return _distribution;
 }
 
+bool
+ContentBucketSpace::getNodeUpInLastNodeStateSeenByProvider() const
+{
+    std::lock_guard guard(_lock);
+    return _nodeUpInLastNodeStateSeenByProvider;
+}
+
+void
+ContentBucketSpace::setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider)
+{
+    std::lock_guard guard(_lock);
+    _nodeUpInLastNodeStateSeenByProvider = nodeUpInLastNodeStateSeenByProvider;
+}
+
 }
diff --git a/storage/src/vespa/storage/common/content_bucket_space.h b/storage/src/vespa/storage/common/content_bucket_space.h
index 6ccc82bc4fb..81ce6234879 100644
--- a/storage/src/vespa/storage/common/content_bucket_space.h
+++ b/storage/src/vespa/storage/common/content_bucket_space.h
@@ -22,6 +22,7 @@ private:
     mutable std::mutex _lock;
     std::shared_ptr<const lib::ClusterState> _clusterState;
     std::shared_ptr<const lib::Distribution> _distribution;
+    bool _nodeUpInLastNodeStateSeenByProvider;
 
 public:
     using UP = std::unique_ptr<ContentBucketSpace>;
@@ -33,6 +34,8 @@ public:
     std::shared_ptr<const lib::ClusterState> getClusterState() const;
     void setDistribution(std::shared_ptr<const lib::Distribution> distribution);
     std::shared_ptr<const lib::Distribution> getDistribution() const;
+    bool getNodeUpInLastNodeStateSeenByProvider() const;
+    void setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider);
 };
 
 }
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
index 961af1532a1..d1f0a24178a 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
@@ -36,7 +36,6 @@ FileStorManager(const config::ConfigUri & configUri, const spi::PartitionStateLi
       _partitions(partitions),
       _providerCore(provider),
       _providerErrorWrapper(_providerCore),
-      _nodeUpInLastNodeStateSeenByProvider(false),
       _providerMetric(new spi::MetricPersistenceProvider(_providerErrorWrapper)),
       _provider(_providerMetric.get()),
       _bucketIdFactory(_component.getBucketIdFactory()),
@@ -886,21 +885,23 @@ FileStorManager::updateState()
     auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle();
     lib::ClusterState::CSP state(clusterStateBundle->getBaselineClusterState());
     lib::Node node(_component.getNodeType(), _component.getIndex());
-    bool nodeUp = state->getNodeState(node).getState().oneOf("uir");
 
     LOG(debug, "FileStorManager received cluster state '%s'", state->toString().c_str());
-    // If edge where we go down
-    if (_nodeUpInLastNodeStateSeenByProvider && !nodeUp) {
-        LOG(debug, "Received cluster state where this node is down; de-activating all buckets in database");
-        Deactivator deactivator;
-        _component.getBucketSpaceRepo().forEachBucket(deactivator, "FileStorManager::updateState");
-    }
     for (const auto &elem : _component.getBucketSpaceRepo()) {
         BucketSpace bucketSpace(elem.first);
-        spi::ClusterState spiState(*elem.second->getClusterState(), _component.getIndex(), *elem.second->getDistribution());
+        ContentBucketSpace &contentBucketSpace = *elem.second;
+        auto derivedClusterState = contentBucketSpace.getClusterState();
+        bool nodeUp = derivedClusterState->getNodeState(node).getState().oneOf("uir");
+        // If edge where we go down
+        if (contentBucketSpace.getNodeUpInLastNodeStateSeenByProvider() && !nodeUp) {
+            LOG(debug, "Received cluster state where this node is down; de-activating all buckets in database for bucket space %s", bucketSpace.toString().c_str());
+            Deactivator deactivator;
+            contentBucketSpace.bucketDatabase().all(deactivator, "FileStorManager::updateState");
+        }
+        contentBucketSpace.setNodeUpInLastNodeStateSeenByProvider(nodeUp);
+        spi::ClusterState spiState(*derivedClusterState, _component.getIndex(), *contentBucketSpace.getDistribution());
         _provider->setClusterState(bucketSpace, spiState);
     }
-    _nodeUpInLastNodeStateSeenByProvider = nodeUp;
 }
 
 void
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
index 0bd2ffa5910..4bf2c1049cf 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
@@ -56,7 +56,6 @@ class FileStorManager : public StorageLinkQueued,
     const spi::PartitionStateList& _partitions;
     spi::PersistenceProvider& _providerCore;
     ProviderErrorWrapper _providerErrorWrapper;
-    bool _nodeUpInLastNodeStateSeenByProvider;
     spi::MetricPersistenceProvider::UP _providerMetric;
     spi::PersistenceProvider* _provider;