Store more variants of ideal service layer nodes for a given bucket in hash table.

12 files changed, 175 insertions, 128 deletions

diff --git a/storage/src/tests/distributor/distributor_bucket_space_test.cpp b/storage/src/tests/distributor/distributor_bucket_space_test.cpp
index e206308618f..8db7955b8a7 100644
--- a/storage/src/tests/distributor/distributor_bucket_space_test.cpp
+++ b/storage/src/tests/distributor/distributor_bucket_space_test.cpp
@@ -16,6 +16,8 @@ namespace {
 
 std::shared_ptr<ClusterState> stable_state(std::make_shared<ClusterState>("distributor:4 storage:4 bits:8"));
 std::shared_ptr<ClusterState> node_1_down_state(std::make_shared<ClusterState>("distributor:4 .1.s:d storage:4 .1.s:d bits:8"));
+std::shared_ptr<ClusterState> node_1_retired_state(std::make_shared<ClusterState>("distributor:4 .1.s:d storage:4 .1.s:r bits:8"));
+std::shared_ptr<ClusterState> node_1_maintenance_state(std::make_shared<ClusterState>("distributor:4 .1.s:d storage:4 .1.s:m bits:8"));
 std::shared_ptr<Distribution> distribution_r1(std::make_shared<Distribution>(Distribution::getDefaultDistributionConfig(1, 4)));
 std::shared_ptr<Distribution> distribution_r2(std::make_shared<Distribution>(Distribution::getDefaultDistributionConfig(2, 4)));
 
@@ -33,70 +35,59 @@ struct DistributorBucketSpaceTest : public ::testing::Test
     {
     }
     ~DistributorBucketSpaceTest() = default;
+
+    // make normal buckets
+    std::vector<BucketId> make_normal_buckets();
+
+    // make deep split buckets. Ideal service layer nodes for a bucket changes for each split level when bucket used bits > 33.
+    std::vector<BucketId> make_deep_split_buckets(std::function<bool(BucketId)> owned);
+
     // Count normal buckets using this distributor
-    uint32_t count_distributor_buckets();
+    uint32_t count_distributor_buckets(const std::vector<BucketId>& buckets);
     // Count normal buckets using service layer node 0.
-    uint32_t count_storage_buckets();
-    // Count deep split buckets using this distributor
-    uint32_t count_deep_split_distributor_buckets();
-    // Count deep split buckets using service layer node 0. Ideal nodes for a bucket changes for each split level when bucket used bits > 33.
-    uint32_t count_deep_split_storage_buckets();
+    CountVector count_service_layer_buckets(const std::vector<BucketId>& buckets);
     // Count normal buckets using this distributor and service layer node 0
     CountVector count_buckets();
     // Count deep split buckets using this distributor and service layer node 0.
     CountVector count_deep_split_buckets();
 };
 
-uint32_t
-DistributorBucketSpaceTest::count_distributor_buckets()
+std::vector<BucketId>
+DistributorBucketSpaceTest::make_normal_buckets()
 {
-    uint32_t owned_buckets = 0;
+    std::vector<BucketId> buckets;
     uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount();
     for (uint32_t i = 0; i < (1u << distribution_bits); ++i) {
-        BucketId bucket(distribution_bits, i);
-        bool owned = bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned();
-        if (owned) {
-            ++owned_buckets;
-        }
+        buckets.emplace_back(distribution_bits, i);
     }
-    return owned_buckets;
+    return buckets;
 }
 
-uint32_t
-DistributorBucketSpaceTest::count_storage_buckets()
+std::vector<BucketId>
+DistributorBucketSpaceTest::make_deep_split_buckets(std::function<bool(BucketId)> owned)
 {
-    uint32_t owned_buckets = 0;
-    uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount();
-    for (uint32_t i = 0; i < (1u << distribution_bits); ++i) {
-        BucketId bucket(distribution_bits, i);
-        auto ideal_nodes = bucket_space.get_ideal_nodes(bucket);
-        auto check_ideal_nodes = bucket_space.get_ideal_nodes_fallback(bucket);
-        EXPECT_EQ(check_ideal_nodes, ideal_nodes);
-        for (auto node : ideal_nodes) {
-            if (node == 0u) {
-                ++owned_buckets;
-            }
-        }
-    }
-    return owned_buckets;
-}
-
-uint32_t
-DistributorBucketSpaceTest::count_deep_split_distributor_buckets()
-{
-    uint32_t owned_buckets = 0;
+    std::vector<BucketId> buckets;
     uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount();
     uint32_t bias = 0;
     uint32_t bias_max = std::min(1u << distribution_bits, 1000u);
     for (; bias < bias_max; ++bias) {
         BucketId bucket(distribution_bits, bias);
-        if (bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned()) {
+        if (owned(bucket)) {
             break;
         }
     }
     assert(bias < bias_max);
     for (uint32_t i = 0; i < 100; ++i) {
-        BucketId bucket(42u, i * (1ul << 32) + bias);
+        buckets.emplace_back(42u, i * (1ul << 32) + bias);
+    }
+    return buckets;
+}
+
+uint32_t
+DistributorBucketSpaceTest::count_distributor_buckets(const std::vector<BucketId>& buckets)
+{
+    uint32_t owned_buckets = 0;
+    for (auto& bucket : buckets) {
         bool owned = bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned();
         if (owned) {
             ++owned_buckets;
@@ -105,47 +96,45 @@ DistributorBucketSpaceTest::count_deep_split_distributor_buckets()
     return owned_buckets;
 }
 
-uint32_t
-DistributorBucketSpaceTest::count_deep_split_storage_buckets()
+DistributorBucketSpaceTest::CountVector
+DistributorBucketSpaceTest::count_service_layer_buckets(const std::vector<BucketId>& buckets)
 {
-    uint32_t owned_buckets = 0;
-    uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount();
-    uint32_t bias = 0;
-    uint32_t bias_max = std::min(1u << distribution_bits, 1000u);
-    for (; bias < bias_max; ++bias) {
-        BucketId bucket(distribution_bits, bias);
-        auto ideal_nodes = bucket_space.get_ideal_nodes(bucket);
-        bool found = false;
-        for (auto node : ideal_nodes) {
-            if (node == 0u) {
-                found = true;
+    CountVector result(3);
+    std::vector<uint16_t> ideal_nodes;
+    for (auto& bucket : buckets) {
+        auto &ideal_nodes_bundle = bucket_space.get_ideal_service_layer_nodes_bundle(bucket);
+        for (uint32_t i = 0; i < 3; ++i) {
+            switch (i) {
+            case 0:
+                ideal_nodes = ideal_nodes_bundle.get_available_nodes();
+                break;
+            case 1:
+                ideal_nodes = ideal_nodes_bundle.get_available_nonretired_nodes();
+                break;
+            case 2:
+                ideal_nodes = ideal_nodes_bundle.get_available_nonretired_or_maintenance_nodes();
+                break;
+            default:
+                ;
             }
-        }
-        if (found) {
-            break;
-        }
-    }
-    assert(bias < bias_max);
-    for (uint32_t i = 0; i < 100; ++i) {
-        BucketId bucket(42u, i * (1ul << 32) + bias);
-        auto ideal_nodes = bucket_space.get_ideal_nodes(bucket);
-        auto check_ideal_nodes = bucket_space.get_ideal_nodes_fallback(bucket);
-        EXPECT_EQ(check_ideal_nodes, ideal_nodes);
-        for (auto node : ideal_nodes) {
-            if (node == 0u) {
-                ++owned_buckets;
+            for (auto node : ideal_nodes) {
+                if (node == 0u) {
+                    ++result[i];
+                }
             }
         }
     }
-    return owned_buckets;
+    return result;
 }
 
 DistributorBucketSpaceTest::CountVector
 DistributorBucketSpaceTest::count_buckets()
 {
     CountVector result;
-    result.push_back(count_distributor_buckets());
-    result.push_back(count_storage_buckets());
+    auto buckets = make_normal_buckets();
+    result.push_back(count_distributor_buckets(buckets));
+    auto service_layer_result = count_service_layer_buckets(buckets);
+    result.insert(result.end(), service_layer_result.cbegin(), service_layer_result.cend());
     return result;
 }
 
@@ -153,8 +142,10 @@ DistributorBucketSpaceTest::CountVector
 DistributorBucketSpaceTest::count_deep_split_buckets()
 {
     CountVector result;
-    result.push_back(count_deep_split_distributor_buckets());
-    result.push_back(count_deep_split_storage_buckets());
+    auto buckets = make_deep_split_buckets([this](BucketId bucket) { return bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned(); });
+    result.push_back(count_distributor_buckets(buckets));
+    auto service_layer_result = count_service_layer_buckets(buckets);
+    result.insert(result.end(), service_layer_result.cbegin(), service_layer_result.cend());
     return result;
 }
 
@@ -162,19 +153,24 @@ TEST_F(DistributorBucketSpaceTest, check_owned_buckets)
 {
     bucket_space.setDistribution(distribution_r1);
     bucket_space.setClusterState(stable_state);
-    EXPECT_EQ((CountVector{64u, 64u}), count_buckets());
+    EXPECT_EQ((CountVector{64u, 64u, 64u, 64u}), count_buckets());
     bucket_space.set_pending_cluster_state(node_1_down_state);
-    EXPECT_EQ((CountVector{64u, 64u}), count_buckets());
+    EXPECT_EQ((CountVector{64u, 64u, 64u, 64u}), count_buckets());
     bucket_space.setClusterState(node_1_down_state);
     bucket_space.set_pending_cluster_state({});
-    EXPECT_EQ((CountVector{86u, 86u}), count_buckets());
+    EXPECT_EQ((CountVector{86u, 86u, 86u, 86u}), count_buckets());
     bucket_space.set_pending_cluster_state(stable_state);
-    EXPECT_EQ((CountVector{64u, 86u}), count_buckets());
+    EXPECT_EQ((CountVector{64u, 86u, 86u, 86u}), count_buckets());
     bucket_space.setClusterState(stable_state);
     bucket_space.set_pending_cluster_state({});
-    EXPECT_EQ((CountVector{64u, 64u}), count_buckets());
+    EXPECT_EQ((CountVector{64u, 64u, 64u, 64u}), count_buckets());
     bucket_space.setDistribution(distribution_r2);
-    EXPECT_EQ((CountVector{64u, 125u}), count_buckets());
+    EXPECT_EQ((CountVector{64u, 125u, 125u, 125u}), count_buckets());
+    bucket_space.setClusterState(node_1_maintenance_state);
+    bucket_space.setDistribution(distribution_r1);
+    EXPECT_EQ((CountVector{86u, 86u, 86u, 64u}), count_buckets());
+    bucket_space.setClusterState(node_1_retired_state);
+    EXPECT_EQ((CountVector{86u, 64u, 86u, 86u}), count_buckets());
 }
 
 TEST_F(DistributorBucketSpaceTest, check_available_nodes)
@@ -198,7 +194,7 @@ TEST_F(DistributorBucketSpaceTest, check_owned_deep_split_buckets)
 {
     bucket_space.setDistribution(distribution_r1);
     bucket_space.setClusterState(stable_state);
-    EXPECT_EQ((CountVector{100u, 19u}), count_deep_split_buckets());
+    EXPECT_EQ((CountVector{100u, 19u, 19u, 19u}), count_deep_split_buckets());
 }
 
 }
diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp
index 3098d8382c8..721809d4515 100644
--- a/storage/src/tests/distributor/operationtargetresolvertest.cpp
+++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp
@@ -120,7 +120,7 @@ OperationTargetResolverTest::getInstances(const BucketId& id,
     idealNodeCalc.setDistribution(distributorBucketSpace.getDistribution());
     idealNodeCalc.setClusterState(distributorBucketSpace.getClusterState());
     OperationTargetResolverImpl resolver(
-            distributorBucketSpace.getBucketDatabase(), idealNodeCalc, 16,
+            distributorBucketSpace, distributorBucketSpace.getBucketDatabase(), 16,
             distributorBucketSpace.getDistribution().getRedundancy(),
             makeBucketSpace());
     if (stripToRedundancy) {
diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp
index 2a220cb9ef8..9838bd9be01 100644
--- a/storage/src/tests/distributor/putoperationtest.cpp
+++ b/storage/src/tests/distributor/putoperationtest.cpp
@@ -484,7 +484,7 @@ PutOperationTest::getNodes(const std::string& infoString) {
 
     std::vector<uint16_t> targetNodes;
     std::vector<uint16_t> createNodes;
-    PutOperation::getTargetNodes(getDistributorBucketSpace().get_ideal_nodes(bid),
+    PutOperation::getTargetNodes(getDistributorBucketSpace().get_ideal_service_layer_nodes_bundle(bid).get_available_nodes(),
                                  targetNodes, createNodes, entry, 2);
 
     ost << "target( ";
diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt
index dd301f0c284..f3ba6af6e0c 100644
--- a/storage/src/vespa/storage/distributor/CMakeLists.txt
+++ b/storage/src/vespa/storage/distributor/CMakeLists.txt
@@ -20,6 +20,7 @@ vespa_add_library(storage_distributor
     externaloperationhandler.cpp
     idealstatemanager.cpp
     idealstatemetricsset.cpp
+    ideal_service_layer_nodes_bundle.cpp
     messagetracker.cpp
     nodeinfo.cpp
     operation_routing_snapshot.cpp
diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp
index f9122650311..07c714cf133 100644
--- a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp
+++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp
@@ -12,6 +12,8 @@ namespace storage::distributor {
 namespace {
 
 const char *up_states = "uri";
+const char *nonretired_up_states = "ui";
+const char *nonretired_or_maintenance_up_states = "uim";
 
 }
 
@@ -79,12 +81,6 @@ DistributorBucketSpace::setDistribution(std::shared_ptr<const lib::Distribution>
     clear();
 }
 
-std::vector<uint16_t>
-DistributorBucketSpace::get_ideal_nodes_fallback(document::BucketId bucket) const
-{
-    return _distribution->getIdealStorageNodes(*_clusterState, bucket, up_states);
-}
-
 void
 DistributorBucketSpace::set_pending_cluster_state(std::shared_ptr<const lib::ClusterState> pending_cluster_state)
 {
@@ -118,20 +114,20 @@ DistributorBucketSpace::owns_bucket_in_state(
     return owns_bucket_in_state(*_distribution, clusterState, bucket);
 }
 
-std::vector<uint16_t>
-DistributorBucketSpace::get_ideal_nodes(document::BucketId bucket) const
+const IdealServiceLayerNodesBundle&
+DistributorBucketSpace::get_ideal_service_layer_nodes_bundle(document::BucketId bucket) const
 {
     assert(bucket.getUsedBits() >= _distribution_bits);
-    if (bucket.getUsedBits() > 33) { // cf. storage::lib::Distribution::getStorageSeed
-        // Cannot map to super bucket ==> cannot cache result
-        return get_ideal_nodes_fallback(bucket);
-    }
-    document::BucketId super_bucket(_distribution_bits, bucket.getId());
-    auto itr = _ideal_nodes.find(super_bucket);
+    document::BucketId lookup_bucket((bucket.getUsedBits() > 33) ? bucket.getUsedBits() : _distribution_bits, bucket.getId());
+    auto itr = _ideal_nodes.find(lookup_bucket);
     if (itr != _ideal_nodes.end()) {
         return itr->second;
     }
-    auto insres = _ideal_nodes.insert(std::make_pair(super_bucket, get_ideal_nodes_fallback(super_bucket)));
+    IdealServiceLayerNodesBundle ideal_nodes_bundle;
+    ideal_nodes_bundle.set_available_nodes(_distribution->getIdealStorageNodes(*_clusterState, bucket, up_states));
+    ideal_nodes_bundle.set_available_nonretired_nodes(_distribution->getIdealStorageNodes(*_clusterState, bucket, nonretired_up_states));
+    ideal_nodes_bundle.set_available_nonretired_or_maintenance_nodes(_distribution->getIdealStorageNodes(*_clusterState, bucket, nonretired_or_maintenance_up_states));
+    auto insres = _ideal_nodes.insert(std::make_pair(lookup_bucket, std::move(ideal_nodes_bundle)));
     assert(insres.second);
     return insres.first->second;
 }
diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.h b/storage/src/vespa/storage/distributor/distributor_bucket_space.h
index b3722ed9c91..f4d5bd6f5aa 100644
--- a/storage/src/vespa/storage/distributor/distributor_bucket_space.h
+++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.h
@@ -3,6 +3,7 @@
 
 #include "bucketownership.h"
 #include "bucket_ownership_flags.h"
+#include "ideal_service_layer_nodes_bundle.h"
 #include <vespa/document/bucket/bucketid.h>
 #include <vespa/vespalib/stllike/hash_map.h>
 #include <memory>
@@ -39,7 +40,7 @@ class DistributorBucketSpace {
     std::shared_ptr<const lib::ClusterState> _pending_cluster_state;
     std::vector<bool>                        _available_nodes;
     mutable vespalib::hash_map<document::BucketId, BucketOwnershipFlags, document::BucketId::hash>  _ownerships;
-    mutable vespalib::hash_map<document::BucketId, std::vector<uint16_t>, document::BucketId::hash> _ideal_nodes;
+    mutable vespalib::hash_map<document::BucketId, IdealServiceLayerNodesBundle, document::BucketId::hash> _ideal_nodes;
 
     void clear();
     void enumerate_available_nodes();
@@ -81,8 +82,6 @@ public:
     void set_pending_cluster_state(std::shared_ptr<const lib::ClusterState> pending_cluster_state);
     const lib::ClusterState& get_pending_cluster_state() const noexcept { return *_pending_cluster_state; }
 
-    std::vector<uint16_t> get_ideal_nodes_fallback(document::BucketId bucket) const;
-
     /**
      * Returns true if this distributor owns the given bucket in the
      * given cluster and current distribution config.
@@ -91,10 +90,11 @@ public:
     bool owns_bucket_in_state(const lib::ClusterState& clusterState, document::BucketId bucket) const;
 
     const std::vector<bool>& get_available_nodes() const { return _available_nodes; }
+
     /**
-     * Returns the ideal nodes for the given bucket.
+     * Returns the ideal nodes bundle for the given bucket.
      */
-    std::vector<uint16_t> get_ideal_nodes(document::BucketId bucket) const;
+    const IdealServiceLayerNodesBundle &get_ideal_service_layer_nodes_bundle(document::BucketId bucket) const;
 
     /*
      * Return bucket ownership flags for the given bucket. Bucket is always
diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.cpp b/storage/src/vespa/storage/distributor/distributorcomponent.cpp
index 86c98cc7b78..d2a2a6fc6a0 100644
--- a/storage/src/vespa/storage/distributor/distributorcomponent.cpp
+++ b/storage/src/vespa/storage/distributor/distributorcomponent.cpp
@@ -225,7 +225,7 @@ DistributorComponent::updateBucketDatabase(
         }
     }
 
-    UpdateBucketDatabaseProcessor processor(getClock(), found_down_node ? up_nodes : changedNodes, bucketSpace.get_ideal_nodes(bucket.getBucketId()), (updateFlags & DatabaseUpdate::RESET_TRUSTED) != 0);
+    UpdateBucketDatabaseProcessor processor(getClock(), found_down_node ? up_nodes : changedNodes, bucketSpace.get_ideal_service_layer_nodes_bundle(bucket.getBucketId()).get_available_nodes(), (updateFlags & DatabaseUpdate::RESET_TRUSTED) != 0);
 
     bucketSpace.getBucketDatabase().process_update(bucket.getBucketId(), processor, (updateFlags & DatabaseUpdate::CREATE_IF_NONEXISTING) != 0);
 }
diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp
new file mode 100644
index 00000000000..069be02eb10
--- /dev/null
+++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp
@@ -0,0 +1,17 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "ideal_service_layer_nodes_bundle.h"
+#include <vespa/vdslib/distribution/idealnodecalculator.h>
+
+namespace storage::distributor {
+
+IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle() noexcept
+  : _available_nodes(),
+    _available_nonretired_nodes(),
+    _available_nonretired_or_maintenance_nodes()
+{
+}
+
+IdealServiceLayerNodesBundle::~IdealServiceLayerNodesBundle() = default;
+
+}
diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h
new file mode 100644
index 00000000000..2fd1bc8ad4b
--- /dev/null
+++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h
@@ -0,0 +1,28 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vector>
+#include <cstdint>
+
+namespace storage::distributor {
+
+/*
+ * Bundle of ideal service layer nodes for a bucket.
+ */
+class IdealServiceLayerNodesBundle {
+    std::vector<uint16_t> _available_nodes;
+    std::vector<uint16_t> _available_nonretired_nodes;
+    std::vector<uint16_t> _available_nonretired_or_maintenance_nodes;
+public:
+    IdealServiceLayerNodesBundle() noexcept;
+    ~IdealServiceLayerNodesBundle();
+
+    void set_available_nodes(std::vector<uint16_t> available_nodes) { _available_nodes = std::move(available_nodes); }
+    void set_available_nonretired_nodes(std::vector<uint16_t> available_nonretired_nodes) { _available_nonretired_nodes = std::move(available_nonretired_nodes); }
+    void set_available_nonretired_or_maintenance_nodes(std::vector<uint16_t> available_nonretired_or_maintenance_nodes) { _available_nonretired_or_maintenance_nodes = std::move(available_nonretired_or_maintenance_nodes); }
+    std::vector<uint16_t> get_available_nodes() const { return _available_nodes; }
+    std::vector<uint16_t> get_available_nonretired_nodes() const { return _available_nonretired_nodes; }
+    std::vector<uint16_t> get_available_nonretired_or_maintenance_nodes() const { return _available_nonretired_or_maintenance_nodes; }
+};
+
+}
diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp
index 8a4f527c43a..7a247f6c524 100644
--- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp
@@ -108,7 +108,7 @@ PutOperation::insertDatabaseEntryAndScheduleCreateBucket(const OperationTargetLi
         (void) multipleBuckets;
         BucketDatabase::Entry entry(_bucketSpace.getBucketDatabase().get(lastBucket));
         std::vector<uint16_t> idealState(
-                _bucketSpace.getDistribution().getIdealStorageNodes(_bucketSpace.getClusterState(), lastBucket, "ui"));
+                _bucketSpace.get_ideal_service_layer_nodes_bundle(lastBucket).get_available_nodes());
         active = ActiveCopy::calculate(idealState, _bucketSpace.getDistribution(), entry);
         LOG(debug, "Active copies for bucket %s: %s", entry.getBucketId().toString().c_str(), active.toString().c_str());
         for (uint32_t i=0; i<active.size(); ++i) {
@@ -182,10 +182,7 @@ PutOperation::onStart(DistributorMessageSender& sender)
     if (up) {
         std::vector<document::BucketId> bucketsToCheckForSplit;
 
-        lib::IdealNodeCalculatorImpl idealNodeCalculator;
-        idealNodeCalculator.setDistribution(_bucketSpace.getDistribution());
-        idealNodeCalculator.setClusterState(_bucketSpace.getClusterState());
-        OperationTargetResolverImpl targetResolver(_bucketSpace.getBucketDatabase(), idealNodeCalculator,
+        OperationTargetResolverImpl targetResolver(_bucketSpace, _bucketSpace.getBucketDatabase(),
                 _op_ctx.distributor_config().getMinimalBucketSplit(),
                 _bucketSpace.getDistribution().getRedundancy(),
                 _msg->getBucket().getBucketSpace());
diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp
index 23bb6b1db78..7e2da056b5a 100644
--- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp
+++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "operationtargetresolverimpl.h"
+#include "distributor_bucket_space.h"
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/util/printable.hpp>
 #include <sstream>
@@ -8,6 +9,20 @@
 
 namespace storage::distributor {
 
+namespace {
+
+lib::IdealNodeList
+make_node_list(const std::vector<uint16_t>& nodes)
+{
+    lib::IdealNodeList list;
+    for (auto node : nodes) {
+        list.push_back(lib::Node(lib::NodeType::STORAGE, node));
+    }
+    return list;
+}
+
+}
+
 BucketInstance::BucketInstance(
         const document::BucketId& id, const api::BucketInfo& info,
         lib::Node node, uint16_t idealLocationPriority, bool trusted, bool exist)
@@ -54,15 +69,12 @@ BucketInstanceList::add(BucketDatabase::Entry& e,
 }
 
 void
-BucketInstanceList::populate(const document::BucketId& specificId, BucketDatabase& db,
-                             const lib::IdealNodeCalculator& idealNodeCalc)
+BucketInstanceList::populate(const document::BucketId& specificId, const DistributorBucketSpace& distributor_bucket_space, BucketDatabase& db)
 {
     std::vector<BucketDatabase::Entry> entries;
     db.getParents(specificId, entries);
     for (uint32_t i=0; i<entries.size(); ++i) {
-        lib::IdealNodeList idealNodes(idealNodeCalc.getIdealStorageNodes(
-                entries[i].getBucketId(),
-                lib::IdealNodeCalculator::UpInitMaintenance));
+        lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(entries[i].getBucketId()).get_available_nonretired_or_maintenance_nodes()));
         add(entries[i], idealNodes);
     }
 }
@@ -108,17 +120,16 @@ BucketInstanceList::leastSpecificLeafBucketInSubtree(
 
 void
 BucketInstanceList::extendToEnoughCopies(
+        const DistributorBucketSpace& distributor_bucket_space,
         const BucketDatabase& db,
         const document::BucketId& targetIfNonPreExisting,
-        const document::BucketId& mostSpecificId,
-        const lib::IdealNodeCalculator& idealNodeCalc)
+        const document::BucketId& mostSpecificId)
 {
     document::BucketId newTarget(_instances.empty() ? targetIfNonPreExisting
                                                     : _instances[0]._bucket);
     newTarget = leastSpecificLeafBucketInSubtree(newTarget, mostSpecificId, db);
 
-    lib::IdealNodeList idealNodes(idealNodeCalc.getIdealStorageNodes(
-            newTarget, lib::IdealNodeCalculator::UpInit));
+    lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(newTarget).get_available_nonretired_nodes()));
     for (uint32_t i=0; i<idealNodes.size(); ++i) {
         if (!contains(idealNodes[i])) {
             _instances.push_back(BucketInstance(
@@ -182,14 +193,14 @@ OperationTargetResolverImpl::getAllInstances(OperationType type,
 {
     BucketInstanceList instances;
     if (type == PUT) {
-        instances.populate(id, _bucketDatabase, _idealNodeCalculator);
+        instances.populate(id, _distributor_bucket_space, _bucketDatabase);
         instances.sort(InstanceOrder());
         instances.removeNodeDuplicates();
         instances.extendToEnoughCopies(
+                _distributor_bucket_space,
                 _bucketDatabase,
                 _bucketDatabase.getAppropriateBucket(_minUsedBucketBits, id),
-                id,
-                _idealNodeCalculator);
+                id);
     } else {
         throw vespalib::IllegalArgumentException(
                 "Unsupported operation type given", VESPA_STRLOC);
diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h
index 73a2c281b18..a23c8ba7f59 100644
--- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h
+++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h
@@ -9,6 +9,8 @@
 
 namespace storage::distributor {
 
+class DistributorBucketSpace;
+
 struct BucketInstance : public vespalib::AsciiPrintable {
     document::BucketId _bucket;
     api::BucketInfo    _info;
@@ -57,13 +59,12 @@ public:
      *   _instances.size() >= configured redundancy level, unless insufficient
      *   number of nodes are available
      */
-    void extendToEnoughCopies(const BucketDatabase& db,
+    void extendToEnoughCopies(const DistributorBucketSpace& distributor_bucket_space,
+                              const BucketDatabase& db,
                               const document::BucketId& targetIfNonPreExisting,
-                              const document::BucketId& mostSpecificId,
-                              const lib::IdealNodeCalculator& idealNodeCalc);
+                              const document::BucketId& mostSpecificId);
 
-    void populate(const document::BucketId&, BucketDatabase&,
-                  const lib::IdealNodeCalculator&);
+    void populate(const document::BucketId&, const DistributorBucketSpace&, BucketDatabase&);
     void add(BucketDatabase::Entry& e, const lib::IdealNodeList& idealState);
 
     template <typename Order>
@@ -77,20 +78,20 @@ public:
 };
 
 class OperationTargetResolverImpl : public OperationTargetResolver {
+    const DistributorBucketSpace& _distributor_bucket_space;
     BucketDatabase& _bucketDatabase;
-    const lib::IdealNodeCalculator& _idealNodeCalculator;
     uint32_t _minUsedBucketBits;
     uint16_t _redundancy;
     document::BucketSpace _bucketSpace;
 
 public:
-    OperationTargetResolverImpl(BucketDatabase& bucketDatabase,
-                                const lib::IdealNodeCalculator& idealNodeCalc,
+    OperationTargetResolverImpl(const DistributorBucketSpace& distributor_bucket_space,
+                                BucketDatabase& bucketDatabase,
                                 uint32_t minUsedBucketBits,
                                 uint16_t redundancy,
                                 document::BucketSpace bucketSpace)
-        : _bucketDatabase(bucketDatabase),
-          _idealNodeCalculator(idealNodeCalc),
+        : _distributor_bucket_space(distributor_bucket_space),
+          _bucketDatabase(bucketDatabase),
           _minUsedBucketBits(minUsedBucketBits),
           _redundancy(redundancy),
           _bucketSpace(bucketSpace)