Add read-only bucket space repo and wire it through distributor components

15 files changed, 129 insertions, 51 deletions

diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp
index b2d554c1e42..bad6e80de47 100644
--- a/storage/src/tests/distributor/bucketdbupdatertest.cpp
+++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp
@@ -112,8 +112,19 @@ class BucketDBUpdaterTest : public CppUnit::TestFixture,
     CPPUNIT_TEST(adding_diverging_replica_to_existing_trusted_does_not_remove_trusted);
     CPPUNIT_TEST(batch_update_from_distributor_change_does_not_mark_diverging_replicas_as_trusted);
     CPPUNIT_TEST(global_distribution_hash_falls_back_to_legacy_format_upon_request_rejection);
+    CPPUNIT_TEST(non_owned_buckets_moved_to_read_only_db_on_ownership_change);
     CPPUNIT_TEST_SUITE_END();
 
+    /*
+     * TODO tests
+     *  - buckets moved to read only db on ownership change
+     *    - even when self is down in pending state
+     *  - buckets NOT moved to read only db on content node down/maintenance
+     *  - read only db cleared when cluster state activated
+     *  - explicit cluster state activation path
+     *  - legacy implicit cluster state activation support
+     */
+
 public:
     BucketDBUpdaterTest();
 
@@ -123,10 +134,7 @@ protected:
     void testDistributorChangeWithGrouping();
     void testNormalUsageInitializing();
     void testFailedRequestBucketInfo();
-    void testNoResponses();
     void testBitChange();
-    void testInconsistentChecksum();
-    void testAddEmptyNode();
     void testNodeDown();
     void testStorageNodeInMaintenanceClearsBucketsForNode();
     void testNodeDownCopiesGetInSync();
@@ -177,6 +185,7 @@ protected:
     void adding_diverging_replica_to_existing_trusted_does_not_remove_trusted();
     void batch_update_from_distributor_change_does_not_mark_diverging_replicas_as_trusted();
     void global_distribution_hash_falls_back_to_legacy_format_upon_request_rejection();
+    void non_owned_buckets_moved_to_read_only_db_on_ownership_change();
 
     auto &defaultDistributorBucketSpace() { return getBucketSpaceRepo().get(makeBucketSpace()); }
 
@@ -190,13 +199,19 @@ protected:
                         getBucketDBUpdater().getDistributorComponent().getIndex(),
                         clusterStateBundle,
                         "ui"));
-        auto &repo = getBucketSpaceRepo();
-        for (auto &elem : repo) {
-            elem.second->setClusterState(clusterStateBundle.getDerivedClusterState(elem.first));
+        for (auto* repo : {&mutable_repo(), &read_only_repo()}) {
+            for (auto& space : *repo) {
+                space.second->setClusterState(clusterStateBundle.getDerivedClusterState(space.first));
+            }
         }
         return clusterInfo;
     }
 
+    DistributorBucketSpaceRepo& mutable_repo() noexcept { return getBucketSpaceRepo(); }
+    // Note: not calling this "immutable_repo" since it may actually be modified by the pending
+    // cluster state component (just not by operations), so it would not have the expected semantics.
+    DistributorBucketSpaceRepo& read_only_repo() noexcept { return getReadOnlyBucketSpaceRepo(); }
+
     static std::string getNodeList(std::vector<uint16_t> nodes, size_t count);
 
     std::string getNodeList(std::vector<uint16_t> nodes);
@@ -586,8 +601,9 @@ public:
 
             OutdatedNodesMap outdatedNodesMap;
             state = PendingClusterState::createForClusterStateChange(
-                    clock, clusterInfo, sender, owner.getBucketSpaceRepo(), cmd, outdatedNodesMap,
-                    api::Timestamp(1));
+                    clock, clusterInfo, sender,
+                    owner.getBucketSpaceRepo(), owner.getReadOnlyBucketSpaceRepo(),
+                    cmd, outdatedNodesMap, api::Timestamp(1));
         }
 
         PendingClusterStateFixture(
@@ -598,7 +614,8 @@ public:
                     owner.createClusterInfo(oldClusterState));
 
             state = PendingClusterState::createForDistributionChange(
-                    clock, clusterInfo, sender, owner.getBucketSpaceRepo(), api::Timestamp(1));
+                    clock, clusterInfo, sender, owner.getBucketSpaceRepo(),
+                    owner.getReadOnlyBucketSpaceRepo(), api::Timestamp(1));
         }
     };
 
@@ -1533,7 +1550,8 @@ BucketDBUpdaterTest::getSentNodesDistributionChanged(
     ClusterInformation::CSP clusterInfo(createClusterInfo(oldClusterState));
     std::unique_ptr<PendingClusterState> state(
             PendingClusterState::createForDistributionChange(
-                    clock, clusterInfo, sender, getBucketSpaceRepo(), api::Timestamp(1)));
+                    clock, clusterInfo, sender, getBucketSpaceRepo(),
+                    getReadOnlyBucketSpaceRepo(), api::Timestamp(1)));
 
     sortSentMessagesByIndex(sender);
 
@@ -1698,8 +1716,8 @@ BucketDBUpdaterTest::testPendingClusterStateReceive()
     OutdatedNodesMap outdatedNodesMap;
     std::unique_ptr<PendingClusterState> state(
             PendingClusterState::createForClusterStateChange(
-                    clock, clusterInfo, sender, getBucketSpaceRepo(), cmd, outdatedNodesMap,
-                    api::Timestamp(1)));
+                    clock, clusterInfo, sender, getBucketSpaceRepo(), getReadOnlyBucketSpaceRepo(),
+                    cmd, outdatedNodesMap, api::Timestamp(1)));
 
     CPPUNIT_ASSERT_EQUAL(messageCount(3), sender.commands.size());
 
@@ -1863,8 +1881,8 @@ BucketDBUpdaterTest::mergeBucketLists(
         ClusterInformation::CSP clusterInfo(createClusterInfo("cluster:d"));
         std::unique_ptr<PendingClusterState> state(
                 PendingClusterState::createForClusterStateChange(
-                        clock, clusterInfo, sender, getBucketSpaceRepo(), cmd, outdatedNodesMap,
-                        beforeTime));
+                        clock, clusterInfo, sender, getBucketSpaceRepo(), getReadOnlyBucketSpaceRepo(),
+                        cmd, outdatedNodesMap, beforeTime));
 
         parseInputData(existingData, beforeTime, *state, includeBucketInfo);
         state->mergeIntoBucketDatabases();
@@ -1882,8 +1900,8 @@ BucketDBUpdaterTest::mergeBucketLists(
         ClusterInformation::CSP clusterInfo(createClusterInfo(oldState.toString()));
         std::unique_ptr<PendingClusterState> state(
                 PendingClusterState::createForClusterStateChange(
-                        clock, clusterInfo, sender, getBucketSpaceRepo(), cmd, outdatedNodesMap,
-                        afterTime));
+                        clock, clusterInfo, sender, getBucketSpaceRepo(), getReadOnlyBucketSpaceRepo(),
+                        cmd, outdatedNodesMap, afterTime));
 
         parseInputData(newData, afterTime, *state, includeBucketInfo);
         state->mergeIntoBucketDatabases();
@@ -2599,4 +2617,8 @@ void BucketDBUpdaterTest::global_distribution_hash_falls_back_to_legacy_format_u
     CPPUNIT_ASSERT_EQUAL(current_hash, new_current_req.getDistributionHash());
 }
 
+void BucketDBUpdaterTest::non_owned_buckets_moved_to_read_only_db_on_ownership_change() {
+
+}
+
 }
diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp
index d3496d0c9f6..3f7f2eac63a 100644
--- a/storage/src/tests/distributor/distributortestutil.cpp
+++ b/storage/src/tests/distributor/distributortestutil.cpp
@@ -388,6 +388,16 @@ DistributorTestUtil::getBucketSpaceRepo() const {
     return _distributor->getBucketSpaceRepo();
 }
 
+DistributorBucketSpaceRepo &
+DistributorTestUtil::getReadOnlyBucketSpaceRepo() {
+    return _distributor->getReadOnlyBucketSpaceRepo();
+}
+
+const DistributorBucketSpaceRepo &
+DistributorTestUtil::getReadOnlyBucketSpaceRepo() const {
+    return _distributor->getReadOnlyBucketSpaceRepo();
+}
+
 const lib::Distribution&
 DistributorTestUtil::getDistribution() const {
     return getBucketSpaceRepo().get(makeBucketSpace()).getDistribution();
diff --git a/storage/src/tests/distributor/distributortestutil.h b/storage/src/tests/distributor/distributortestutil.h
index 10cc5eeaca1..420111437d2 100644
--- a/storage/src/tests/distributor/distributortestutil.h
+++ b/storage/src/tests/distributor/distributortestutil.h
@@ -132,6 +132,8 @@ public:
     const BucketDatabase& getBucketDatabase(document::BucketSpace space) const;
     DistributorBucketSpaceRepo &getBucketSpaceRepo();
     const DistributorBucketSpaceRepo &getBucketSpaceRepo() const;
+    DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo();
+    const DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo() const;
 
     const lib::Distribution& getDistribution() const;
     // "End to end" distribution change trigger, which will invoke the bucket
diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
index a223001af79..97028c20191 100644
--- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
+++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
@@ -20,11 +20,12 @@ using document::BucketSpace;
 namespace storage::distributor {
 
 BucketDBUpdater::BucketDBUpdater(Distributor& owner,
-                                 DistributorBucketSpaceRepo &bucketSpaceRepo,
+                                 DistributorBucketSpaceRepo& bucketSpaceRepo,
+                                 DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
                                  DistributorMessageSender& sender,
                                  DistributorComponentRegister& compReg)
     : framework::StatusReporter("bucketdb", "Bucket DB Updater"),
-      _distributorComponent(owner, bucketSpaceRepo, compReg, "Bucket DB Updater"),
+      _distributorComponent(owner, bucketSpaceRepo, readOnlyBucketSpaceRepo, compReg, "Bucket DB Updater"),
       _sender(sender),
       _transitionTimer(_distributorComponent.getClock())
 {
@@ -169,6 +170,7 @@ BucketDBUpdater::storageDistributionChanged()
             std::move(clusterInfo),
             _sender,
             _distributorComponent.getBucketSpaceRepo(),
+            _distributorComponent.getReadOnlyBucketSpaceRepo(),
             _distributorComponent.getUniqueTimestamp());
     _outdatedNodesMap = _pendingClusterState->getOutdatedNodesMap();
 }
@@ -200,6 +202,7 @@ BucketDBUpdater::onSetSystemState(
     }
     ensureTransitionTimerStarted();
 
+    // TODO
     removeSuperfluousBuckets(cmd->getClusterStateBundle());
     replyToPreviousPendingClusterStateIfAny();
 
@@ -214,6 +217,7 @@ BucketDBUpdater::onSetSystemState(
             std::move(clusterInfo),
             _sender,
             _distributorComponent.getBucketSpaceRepo(),
+            _distributorComponent.getReadOnlyBucketSpaceRepo(),
             cmd,
             _outdatedNodesMap,
             _distributorComponent.getUniqueTimestamp());
diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h
index ea67e7ea72a..a31b62d4e9b 100644
--- a/storage/src/vespa/storage/distributor/bucketdbupdater.h
+++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h
@@ -33,7 +33,8 @@ public:
     using OutdatedNodes = dbtransition::OutdatedNodes;
     using OutdatedNodesMap = dbtransition::OutdatedNodesMap;
     BucketDBUpdater(Distributor& owner,
-                    DistributorBucketSpaceRepo &bucketSpaceRepo,
+                    DistributorBucketSpaceRepo& bucketSpaceRepo,
+                    DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
                     DistributorMessageSender& sender,
                     DistributorComponentRegister& compReg);
     ~BucketDBUpdater();
diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp
index 1664dd0d9a1..6184f8ff26a 100644
--- a/storage/src/vespa/storage/distributor/distributor.cpp
+++ b/storage/src/vespa/storage/distributor/distributor.cpp
@@ -67,15 +67,16 @@ Distributor::Distributor(DistributorComponentRegister& compReg,
       _compReg(compReg),
       _component(compReg, "distributor"),
       _bucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>()),
+      _readOnlyBucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>()),
       _metrics(new DistributorMetricSet(_component.getLoadTypes()->getMetricLoadTypes())),
       _operationOwner(*this, _component.getClock()),
       _maintenanceOperationOwner(*this, _component.getClock()),
       _pendingMessageTracker(compReg),
-      _bucketDBUpdater(*this, *_bucketSpaceRepo, *this, compReg),
+      _bucketDBUpdater(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, *this, compReg),
       _distributorStatusDelegate(compReg, *this, *this),
       _bucketDBStatusDelegate(compReg, *this, _bucketDBUpdater),
-      _idealStateManager(*this, *_bucketSpaceRepo, compReg, manageActiveBucketCopies),
-      _externalOperationHandler(*this, *_bucketSpaceRepo, _idealStateManager, compReg),
+      _idealStateManager(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, compReg, manageActiveBucketCopies),
+      _externalOperationHandler(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, _idealStateManager, compReg),
       _threadPool(threadPool),
       _initializingIsUp(true),
       _doneInitializeHandler(doneInitHandler),
@@ -575,16 +576,20 @@ void
 Distributor::propagateDefaultDistribution(
         std::shared_ptr<const lib::Distribution> distribution)
 {
-    _bucketSpaceRepo->get(document::FixedBucketSpaces::default_space()).setDistribution(distribution);
-    auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution);
-    _bucketSpaceRepo->get(document::FixedBucketSpaces::global_space()).setDistribution(std::move(global_distr));
+    for (auto* repo : {_bucketSpaceRepo.get(), _readOnlyBucketSpaceRepo.get()}) {
+        repo->get(document::FixedBucketSpaces::default_space()).setDistribution(distribution);
+        auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution);
+        repo->get(document::FixedBucketSpaces::global_space()).setDistribution(std::move(global_distr));
+    }
 }
 
 void
 Distributor::propagateClusterStates()
 {
-    for (auto &iter : *_bucketSpaceRepo) {
-        iter.second->setClusterState(_clusterStateBundle.getDerivedClusterState(iter.first));
+    for (auto* repo : {_bucketSpaceRepo.get(), _readOnlyBucketSpaceRepo.get()}) {
+        for (auto& iter : *repo) {
+            iter.second->setClusterState(_clusterStateBundle.getDerivedClusterState(iter.first));
+        }
     }
 }
 
diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h
index fb8a9fb4299..cd24b91eba2 100644
--- a/storage/src/vespa/storage/distributor/distributor.h
+++ b/storage/src/vespa/storage/distributor/distributor.h
@@ -158,6 +158,13 @@ public:
     DistributorBucketSpaceRepo &getBucketSpaceRepo() noexcept { return *_bucketSpaceRepo; }
     const DistributorBucketSpaceRepo &getBucketSpaceRepo() const noexcept { return *_bucketSpaceRepo; }
 
+    DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo() noexcept {
+        return *_readOnlyBucketSpaceRepo;
+    }
+    const DistributorBucketSpaceRepo& getReadyOnlyBucketSpaceRepo() const noexcept {
+        return *_readOnlyBucketSpaceRepo;
+    }
+
 private:
     friend class Distributor_Test;
     friend class BucketDBUpdaterTest;
@@ -244,6 +251,10 @@ private:
     DistributorComponentRegister& _compReg;
     storage::DistributorComponent _component;
     std::unique_ptr<DistributorBucketSpaceRepo> _bucketSpaceRepo;
+    // Read-only bucket space repo with DBs that only contain buckets transiently
+    // during cluster state transitions. Bucket set does not overlap that of _bucketSpaceRepo
+    // and the DBs are empty during non-transition phases.
+    std::unique_ptr<DistributorBucketSpaceRepo> _readOnlyBucketSpaceRepo;
     std::shared_ptr<DistributorMetricSet> _metrics;
 
     OperationOwner _operationOwner;
diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.cpp b/storage/src/vespa/storage/distributor/distributorcomponent.cpp
index d3d07350d35..9bd215b9644 100644
--- a/storage/src/vespa/storage/distributor/distributorcomponent.cpp
+++ b/storage/src/vespa/storage/distributor/distributorcomponent.cpp
@@ -15,16 +15,18 @@ namespace storage::distributor {
 
 DistributorComponent::DistributorComponent(
         DistributorInterface& distributor,
-        DistributorBucketSpaceRepo &bucketSpaceRepo,
+        DistributorBucketSpaceRepo& bucketSpaceRepo,
+        DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
         DistributorComponentRegister& compReg,
         const std::string& name)
     : storage::DistributorComponent(compReg, name),
       _distributor(distributor),
-      _bucketSpaceRepo(bucketSpaceRepo)
+      _bucketSpaceRepo(bucketSpaceRepo),
+      _readOnlyBucketSpaceRepo(readOnlyBucketSpaceRepo)
 {
 }
 
-DistributorComponent::~DistributorComponent() {}
+DistributorComponent::~DistributorComponent() = default;
 
 void
 DistributorComponent::sendDown(const api::StorageMessage::SP& msg)
diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.h b/storage/src/vespa/storage/distributor/distributorcomponent.h
index 561904cee8d..f2aea89d47c 100644
--- a/storage/src/vespa/storage/distributor/distributorcomponent.h
+++ b/storage/src/vespa/storage/distributor/distributorcomponent.h
@@ -29,11 +29,12 @@ class DistributorComponent : public storage::DistributorComponent
 {
 public:
     DistributorComponent(DistributorInterface& distributor,
-                         DistributorBucketSpaceRepo &bucketSpaceRepo,
-  		         DistributorComponentRegister& compReg,
-		         const std::string& name);
+                         DistributorBucketSpaceRepo& bucketSpaceRepo,
+                         DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
+                         DistributorComponentRegister& compReg,
+                         const std::string& name);
 
-    ~DistributorComponent();
+    ~DistributorComponent() override;
 
     /**
      * Returns the ownership status of a bucket as decided with the given
@@ -153,6 +154,9 @@ public:
     DistributorBucketSpaceRepo &getBucketSpaceRepo() { return _bucketSpaceRepo; }
     const DistributorBucketSpaceRepo &getBucketSpaceRepo() const { return _bucketSpaceRepo; }
 
+    DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo() { return _readOnlyBucketSpaceRepo; }
+    const DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo() const { return _readOnlyBucketSpaceRepo; }
+
     /**
      * Finds a bucket that has the same direct parent as the given bucket
      * (i.e. split one bit less), but different bit in the most used bit.
@@ -179,7 +183,8 @@ private:
 
 protected:
 
-    DistributorBucketSpaceRepo &_bucketSpaceRepo;
+    DistributorBucketSpaceRepo& _bucketSpaceRepo;
+    DistributorBucketSpaceRepo& _readOnlyBucketSpaceRepo;
     vespalib::Lock _sync;
 };
 
diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp
index b22592af327..447a5bb156c 100644
--- a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp
+++ b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp
@@ -24,10 +24,12 @@ LOG_SETUP(".distributor.manager");
 
 namespace storage::distributor {
 
-ExternalOperationHandler::ExternalOperationHandler(Distributor& owner, DistributorBucketSpaceRepo& bucketSpaceRepo,
+ExternalOperationHandler::ExternalOperationHandler(Distributor& owner,
+                                                   DistributorBucketSpaceRepo& bucketSpaceRepo,
+                                                   DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
                                                    const MaintenanceOperationGenerator& gen,
                                                    DistributorComponentRegister& compReg)
-    : DistributorComponent(owner, bucketSpaceRepo, compReg, "External operation handler"),
+    : DistributorComponent(owner, bucketSpaceRepo, readOnlyBucketSpaceRepo, compReg, "External operation handler"),
       _operationGenerator(gen),
       _rejectFeedBeforeTimeReached() // At epoch
 { }
diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h
index c198fe30159..67fff2a9f39 100644
--- a/storage/src/vespa/storage/distributor/externaloperationhandler.h
+++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h
@@ -37,6 +37,7 @@ public:
 
     ExternalOperationHandler(Distributor& owner,
                              DistributorBucketSpaceRepo& bucketSpaceRepo,
+                             DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
                              const MaintenanceOperationGenerator&,
                              DistributorComponentRegister& compReg);
 
diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp
index 77b924ad351..5a1ff31e2e7 100644
--- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp
+++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp
@@ -26,11 +26,12 @@ namespace distributor {
 IdealStateManager::IdealStateManager(
         Distributor& owner,
         DistributorBucketSpaceRepo& bucketSpaceRepo,
+        DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
         DistributorComponentRegister& compReg,
         bool manageActiveBucketCopies)
     : HtmlStatusReporter("idealstateman", "Ideal state manager"),
       _metrics(new IdealStateMetricSet),
-      _distributorComponent(owner, bucketSpaceRepo, compReg, "Ideal state manager"),
+      _distributorComponent(owner, bucketSpaceRepo, readOnlyBucketSpaceRepo, compReg, "Ideal state manager"),
       _bucketSpaceRepo(bucketSpaceRepo)
 {
     _distributorComponent.registerStatusPage(*this);
diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.h b/storage/src/vespa/storage/distributor/idealstatemanager.h
index c8be2a40ad7..3bb6d0dd757 100644
--- a/storage/src/vespa/storage/distributor/idealstatemanager.h
+++ b/storage/src/vespa/storage/distributor/idealstatemanager.h
@@ -37,6 +37,7 @@ public:
 
     IdealStateManager(Distributor& owner,
                       DistributorBucketSpaceRepo& bucketSpaceRepo,
+                      DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
                       DistributorComponentRegister& compReg,
                       bool manageActiveBucketCopies);
 
diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp
index 5f74a82c28a..ddbf87d4f24 100644
--- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp
+++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp
@@ -27,7 +27,8 @@ PendingClusterState::PendingClusterState(
         const framework::Clock& clock,
         const ClusterInformation::CSP& clusterInfo,
         DistributorMessageSender& sender,
-        DistributorBucketSpaceRepo &bucketSpaceRepo,
+        DistributorBucketSpaceRepo& bucketSpaceRepo,
+        DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
         const std::shared_ptr<api::SetSystemStateCommand>& newStateCmd,
         const OutdatedNodesMap &outdatedNodesMap,
         api::Timestamp creationTimestamp)
@@ -40,6 +41,7 @@ PendingClusterState::PendingClusterState(
       _creationTimestamp(creationTimestamp),
       _sender(sender),
       _bucketSpaceRepo(bucketSpaceRepo),
+      _readOnlyBucketSpaceRepo(readOnlyBucketSpaceRepo),
       _bucketOwnershipTransfer(false),
       _pendingTransitions()
 {
@@ -51,7 +53,8 @@ PendingClusterState::PendingClusterState(
         const framework::Clock& clock,
         const ClusterInformation::CSP& clusterInfo,
         DistributorMessageSender& sender,
-        DistributorBucketSpaceRepo &bucketSpaceRepo,
+        DistributorBucketSpaceRepo& bucketSpaceRepo,
+        DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
         api::Timestamp creationTimestamp)
     : _requestedNodes(clusterInfo->getStorageNodeCount()),
       _prevClusterStateBundle(clusterInfo->getClusterStateBundle()),
@@ -61,6 +64,7 @@ PendingClusterState::PendingClusterState(
       _creationTimestamp(creationTimestamp),
       _sender(sender),
       _bucketSpaceRepo(bucketSpaceRepo),
+      _readOnlyBucketSpaceRepo(readOnlyBucketSpaceRepo),
       _bucketOwnershipTransfer(true),
       _pendingTransitions()
 {
diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.h b/storage/src/vespa/storage/distributor/pendingclusterstate.h
index b96ba8cbbd7..cfccf36a61e 100644
--- a/storage/src/vespa/storage/distributor/pendingclusterstate.h
+++ b/storage/src/vespa/storage/distributor/pendingclusterstate.h
@@ -45,15 +45,16 @@ public:
             const framework::Clock& clock,
             const ClusterInformation::CSP& clusterInfo,
             DistributorMessageSender& sender,
-            DistributorBucketSpaceRepo &bucketSpaceRepo,
+            DistributorBucketSpaceRepo& bucketSpaceRepo,
+            DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
             const std::shared_ptr<api::SetSystemStateCommand>& newStateCmd,
             const OutdatedNodesMap &outdatedNodesMap,
             api::Timestamp creationTimestamp)
     {
-        return std::unique_ptr<PendingClusterState>(
-                new PendingClusterState(clock, clusterInfo, sender, bucketSpaceRepo, newStateCmd,
-                                        outdatedNodesMap,
-                                        creationTimestamp));
+        // Naked new due to private constructor
+        return std::unique_ptr<PendingClusterState>(new PendingClusterState(
+                clock, clusterInfo, sender, bucketSpaceRepo, readOnlyBucketSpaceRepo,
+                newStateCmd, outdatedNodesMap, creationTimestamp));
     }
 
     /**
@@ -64,16 +65,19 @@ public:
             const framework::Clock& clock,
             const ClusterInformation::CSP& clusterInfo,
             DistributorMessageSender& sender,
-            DistributorBucketSpaceRepo &bucketSpaceRepo,
+            DistributorBucketSpaceRepo& bucketSpaceRepo,
+            DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
             api::Timestamp creationTimestamp)
     {
-        return std::unique_ptr<PendingClusterState>(
-                new PendingClusterState(clock, clusterInfo, sender, bucketSpaceRepo, creationTimestamp));
+        // Naked new due to private constructor
+        return std::unique_ptr<PendingClusterState>(new PendingClusterState(
+                clock, clusterInfo, sender, bucketSpaceRepo,
+                readOnlyBucketSpaceRepo, creationTimestamp));
     }
 
     PendingClusterState(const PendingClusterState &) = delete;
     PendingClusterState & operator = (const PendingClusterState &) = delete;
-    ~PendingClusterState();
+    ~PendingClusterState() override;
 
     /**
      * Adds the info from the reply to our list of information.
@@ -141,7 +145,8 @@ private:
             const framework::Clock&,
             const ClusterInformation::CSP& clusterInfo,
             DistributorMessageSender& sender,
-            DistributorBucketSpaceRepo &bucketSpaceRepo,
+            DistributorBucketSpaceRepo& bucketSpaceRepo,
+            DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
             const std::shared_ptr<api::SetSystemStateCommand>& newStateCmd,
             const OutdatedNodesMap &outdatedNodesMap,
             api::Timestamp creationTimestamp);
@@ -154,7 +159,8 @@ private:
             const framework::Clock&,
             const ClusterInformation::CSP& clusterInfo,
             DistributorMessageSender& sender,
-            DistributorBucketSpaceRepo &bucketSpaceRepo,
+            DistributorBucketSpaceRepo& bucketSpaceRepo,
+            DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo,
             api::Timestamp creationTimestamp);
 
     struct BucketSpaceAndNode {
@@ -205,6 +211,7 @@ private:
 
     DistributorMessageSender& _sender;
     DistributorBucketSpaceRepo &_bucketSpaceRepo;
+    DistributorBucketSpaceRepo &_readOnlyBucketSpaceRepo;
 
     bool _bucketOwnershipTransfer;
     std::unordered_map<document::BucketSpace, std::unique_ptr<PendingBucketSpaceDbTransition>, document::BucketSpace::hash> _pendingTransitions;