diff options
27 files changed, 263 insertions, 140 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java index 533fc36aad3..f837695d65e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java @@ -10,7 +10,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.component.Handler; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class MbusClient extends Handler<AbstractConfigProducer<?>> implements SessionConfig.Producer { private static final ComponentSpecification CLASSNAME = diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 824a374642a..117ac9bbf4b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2206,6 +2206,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private ZoneId requireZone(String environment, String region) { ZoneId zone = ZoneId.from(environment, region); + // TODO(mpolden): Find a way to not hardcode this. Some APIs allow this "virtual" zone, e.g. /logs + if (zone.environment() == Environment.prod && zone.region().value().equals("controller")) { + return zone; + } if (!controller.zoneRegistry().hasZone(zone)) { throw new IllegalArgumentException("Zone " + zone + " does not exist in this system"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 27d7b6f3d7a..3d1375601ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -494,6 +494,11 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "INFO - All good"); + // GET controller logs + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/environment/prod/region/controller/instance/default/logs?from=1233&to=3214", GET) + .userIdentity(USER_ID), + "INFO - All good"); + // Get content - root tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/content/", GET).userIdentity(USER_ID), "{\"path\":\"/\"}"); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 8662dbc2bfe..f1eaf522308 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -266,7 +266,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag ENABLE_AUTOMATIC_REINDEXING = defineFeatureFlag( - "enable-automatic-reindexing", false, + "enable-automatic-reindexing", true, List.of("bjorncs", "jonmv"), "2020-12-02", "2021-02-01", "Whether to automatically trigger reindexing from config change", "Takes effect on next internal redeployment", diff --git a/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java b/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java index 309316b450a..ca772ce6c3a 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java @@ -4,7 +4,7 @@ package com.yahoo.messagebus; import java.util.concurrent.atomic.AtomicBoolean; /** - * A session which supports receiving, forwarding and acknowledgement of messages. An intermediate session is expacted + * A session which supports receiving, forwarding and acknowledging messages. An intermediate session is expected * to either forward or acknowledge every message received. * * @author Simon Thoresen Hult @@ -49,7 +49,7 @@ public final class IntermediateSession implements MessageHandler, ReplyHandler { /** * This method unregisters this session from message bus, effectively disabling any more messages from being * delivered to the message handler. After unregistering, this method calls {@link com.yahoo.messagebus.MessageBus#sync()} - * as to ensure that there are no threads currently entangled in the handler. + * to ensure that there are no threads currently entangled in the handler. * * This method will deadlock if you call it from the message or reply handler. */ diff --git a/messagebus/src/main/java/com/yahoo/messagebus/SourceSession.java b/messagebus/src/main/java/com/yahoo/messagebus/SourceSession.java index 0b731a4c6fb..204bf2f7906 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/SourceSession.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/SourceSession.java @@ -186,7 +186,7 @@ public final class SourceSession implements ReplyHandler, MessageBus.SendBlocked if (msg.isExpired()) { Error error = new Error(ErrorCode.TIMEOUT, "Timed out in sendQ"); notifyComplete(new Result(error)); - replyHandler.handleReply(createSendTimedoutReply(msg, error)); + replyHandler.handleReply(createSendTimedOutReply(msg, error)); return true; } return false; @@ -214,7 +214,7 @@ public final class SourceSession implements ReplyHandler, MessageBus.SendBlocked } } - private Reply createSendTimedoutReply(Message msg, Error error) { + private Reply createSendTimedOutReply(Message msg, Error error) { Reply reply = new EmptyReply(); reply.setMessage(msg); reply.addError(error); diff --git a/storage/src/tests/distributor/distributor_bucket_space_test.cpp b/storage/src/tests/distributor/distributor_bucket_space_test.cpp index e206308618f..8db7955b8a7 100644 --- a/storage/src/tests/distributor/distributor_bucket_space_test.cpp +++ b/storage/src/tests/distributor/distributor_bucket_space_test.cpp @@ -16,6 +16,8 @@ namespace { std::shared_ptr<ClusterState> stable_state(std::make_shared<ClusterState>("distributor:4 storage:4 bits:8")); std::shared_ptr<ClusterState> node_1_down_state(std::make_shared<ClusterState>("distributor:4 .1.s:d storage:4 .1.s:d bits:8")); +std::shared_ptr<ClusterState> node_1_retired_state(std::make_shared<ClusterState>("distributor:4 .1.s:d storage:4 .1.s:r bits:8")); +std::shared_ptr<ClusterState> node_1_maintenance_state(std::make_shared<ClusterState>("distributor:4 .1.s:d storage:4 .1.s:m bits:8")); std::shared_ptr<Distribution> distribution_r1(std::make_shared<Distribution>(Distribution::getDefaultDistributionConfig(1, 4))); std::shared_ptr<Distribution> distribution_r2(std::make_shared<Distribution>(Distribution::getDefaultDistributionConfig(2, 4))); @@ -33,70 +35,59 @@ struct DistributorBucketSpaceTest : public ::testing::Test { } ~DistributorBucketSpaceTest() = default; + + // make normal buckets + std::vector<BucketId> make_normal_buckets(); + + // make deep split buckets. Ideal service layer nodes for a bucket changes for each split level when bucket used bits > 33. + std::vector<BucketId> make_deep_split_buckets(std::function<bool(BucketId)> owned); + // Count normal buckets using this distributor - uint32_t count_distributor_buckets(); + uint32_t count_distributor_buckets(const std::vector<BucketId>& buckets); // Count normal buckets using service layer node 0. - uint32_t count_storage_buckets(); - // Count deep split buckets using this distributor - uint32_t count_deep_split_distributor_buckets(); - // Count deep split buckets using service layer node 0. Ideal nodes for a bucket changes for each split level when bucket used bits > 33. - uint32_t count_deep_split_storage_buckets(); + CountVector count_service_layer_buckets(const std::vector<BucketId>& buckets); // Count normal buckets using this distributor and service layer node 0 CountVector count_buckets(); // Count deep split buckets using this distributor and service layer node 0. CountVector count_deep_split_buckets(); }; -uint32_t -DistributorBucketSpaceTest::count_distributor_buckets() +std::vector<BucketId> +DistributorBucketSpaceTest::make_normal_buckets() { - uint32_t owned_buckets = 0; + std::vector<BucketId> buckets; uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount(); for (uint32_t i = 0; i < (1u << distribution_bits); ++i) { - BucketId bucket(distribution_bits, i); - bool owned = bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned(); - if (owned) { - ++owned_buckets; - } + buckets.emplace_back(distribution_bits, i); } - return owned_buckets; + return buckets; } -uint32_t -DistributorBucketSpaceTest::count_storage_buckets() +std::vector<BucketId> +DistributorBucketSpaceTest::make_deep_split_buckets(std::function<bool(BucketId)> owned) { - uint32_t owned_buckets = 0; - uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount(); - for (uint32_t i = 0; i < (1u << distribution_bits); ++i) { - BucketId bucket(distribution_bits, i); - auto ideal_nodes = bucket_space.get_ideal_nodes(bucket); - auto check_ideal_nodes = bucket_space.get_ideal_nodes_fallback(bucket); - EXPECT_EQ(check_ideal_nodes, ideal_nodes); - for (auto node : ideal_nodes) { - if (node == 0u) { - ++owned_buckets; - } - } - } - return owned_buckets; -} - -uint32_t -DistributorBucketSpaceTest::count_deep_split_distributor_buckets() -{ - uint32_t owned_buckets = 0; + std::vector<BucketId> buckets; uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount(); uint32_t bias = 0; uint32_t bias_max = std::min(1u << distribution_bits, 1000u); for (; bias < bias_max; ++bias) { BucketId bucket(distribution_bits, bias); - if (bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned()) { + if (owned(bucket)) { break; } } assert(bias < bias_max); for (uint32_t i = 0; i < 100; ++i) { - BucketId bucket(42u, i * (1ul << 32) + bias); + buckets.emplace_back(42u, i * (1ul << 32) + bias); + } + return buckets; +} + +uint32_t +DistributorBucketSpaceTest::count_distributor_buckets(const std::vector<BucketId>& buckets) +{ + uint32_t owned_buckets = 0; + for (auto& bucket : buckets) { bool owned = bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned(); if (owned) { ++owned_buckets; @@ -105,47 +96,45 @@ DistributorBucketSpaceTest::count_deep_split_distributor_buckets() return owned_buckets; } -uint32_t -DistributorBucketSpaceTest::count_deep_split_storage_buckets() +DistributorBucketSpaceTest::CountVector +DistributorBucketSpaceTest::count_service_layer_buckets(const std::vector<BucketId>& buckets) { - uint32_t owned_buckets = 0; - uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount(); - uint32_t bias = 0; - uint32_t bias_max = std::min(1u << distribution_bits, 1000u); - for (; bias < bias_max; ++bias) { - BucketId bucket(distribution_bits, bias); - auto ideal_nodes = bucket_space.get_ideal_nodes(bucket); - bool found = false; - for (auto node : ideal_nodes) { - if (node == 0u) { - found = true; + CountVector result(3); + std::vector<uint16_t> ideal_nodes; + for (auto& bucket : buckets) { + auto &ideal_nodes_bundle = bucket_space.get_ideal_service_layer_nodes_bundle(bucket); + for (uint32_t i = 0; i < 3; ++i) { + switch (i) { + case 0: + ideal_nodes = ideal_nodes_bundle.get_available_nodes(); + break; + case 1: + ideal_nodes = ideal_nodes_bundle.get_available_nonretired_nodes(); + break; + case 2: + ideal_nodes = ideal_nodes_bundle.get_available_nonretired_or_maintenance_nodes(); + break; + default: + ; } - } - if (found) { - break; - } - } - assert(bias < bias_max); - for (uint32_t i = 0; i < 100; ++i) { - BucketId bucket(42u, i * (1ul << 32) + bias); - auto ideal_nodes = bucket_space.get_ideal_nodes(bucket); - auto check_ideal_nodes = bucket_space.get_ideal_nodes_fallback(bucket); - EXPECT_EQ(check_ideal_nodes, ideal_nodes); - for (auto node : ideal_nodes) { - if (node == 0u) { - ++owned_buckets; + for (auto node : ideal_nodes) { + if (node == 0u) { + ++result[i]; + } } } } - return owned_buckets; + return result; } DistributorBucketSpaceTest::CountVector DistributorBucketSpaceTest::count_buckets() { CountVector result; - result.push_back(count_distributor_buckets()); - result.push_back(count_storage_buckets()); + auto buckets = make_normal_buckets(); + result.push_back(count_distributor_buckets(buckets)); + auto service_layer_result = count_service_layer_buckets(buckets); + result.insert(result.end(), service_layer_result.cbegin(), service_layer_result.cend()); return result; } @@ -153,8 +142,10 @@ DistributorBucketSpaceTest::CountVector DistributorBucketSpaceTest::count_deep_split_buckets() { CountVector result; - result.push_back(count_deep_split_distributor_buckets()); - result.push_back(count_deep_split_storage_buckets()); + auto buckets = make_deep_split_buckets([this](BucketId bucket) { return bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned(); }); + result.push_back(count_distributor_buckets(buckets)); + auto service_layer_result = count_service_layer_buckets(buckets); + result.insert(result.end(), service_layer_result.cbegin(), service_layer_result.cend()); return result; } @@ -162,19 +153,24 @@ TEST_F(DistributorBucketSpaceTest, check_owned_buckets) { bucket_space.setDistribution(distribution_r1); bucket_space.setClusterState(stable_state); - EXPECT_EQ((CountVector{64u, 64u}), count_buckets()); + EXPECT_EQ((CountVector{64u, 64u, 64u, 64u}), count_buckets()); bucket_space.set_pending_cluster_state(node_1_down_state); - EXPECT_EQ((CountVector{64u, 64u}), count_buckets()); + EXPECT_EQ((CountVector{64u, 64u, 64u, 64u}), count_buckets()); bucket_space.setClusterState(node_1_down_state); bucket_space.set_pending_cluster_state({}); - EXPECT_EQ((CountVector{86u, 86u}), count_buckets()); + EXPECT_EQ((CountVector{86u, 86u, 86u, 86u}), count_buckets()); bucket_space.set_pending_cluster_state(stable_state); - EXPECT_EQ((CountVector{64u, 86u}), count_buckets()); + EXPECT_EQ((CountVector{64u, 86u, 86u, 86u}), count_buckets()); bucket_space.setClusterState(stable_state); bucket_space.set_pending_cluster_state({}); - EXPECT_EQ((CountVector{64u, 64u}), count_buckets()); + EXPECT_EQ((CountVector{64u, 64u, 64u, 64u}), count_buckets()); bucket_space.setDistribution(distribution_r2); - EXPECT_EQ((CountVector{64u, 125u}), count_buckets()); + EXPECT_EQ((CountVector{64u, 125u, 125u, 125u}), count_buckets()); + bucket_space.setClusterState(node_1_maintenance_state); + bucket_space.setDistribution(distribution_r1); + EXPECT_EQ((CountVector{86u, 86u, 86u, 64u}), count_buckets()); + bucket_space.setClusterState(node_1_retired_state); + EXPECT_EQ((CountVector{86u, 64u, 86u, 86u}), count_buckets()); } TEST_F(DistributorBucketSpaceTest, check_available_nodes) @@ -198,7 +194,7 @@ TEST_F(DistributorBucketSpaceTest, check_owned_deep_split_buckets) { bucket_space.setDistribution(distribution_r1); bucket_space.setClusterState(stable_state); - EXPECT_EQ((CountVector{100u, 19u}), count_deep_split_buckets()); + EXPECT_EQ((CountVector{100u, 19u, 19u, 19u}), count_deep_split_buckets()); } } diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp index 3098d8382c8..721809d4515 100644 --- a/storage/src/tests/distributor/operationtargetresolvertest.cpp +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -120,7 +120,7 @@ OperationTargetResolverTest::getInstances(const BucketId& id, idealNodeCalc.setDistribution(distributorBucketSpace.getDistribution()); idealNodeCalc.setClusterState(distributorBucketSpace.getClusterState()); OperationTargetResolverImpl resolver( - distributorBucketSpace.getBucketDatabase(), idealNodeCalc, 16, + distributorBucketSpace, distributorBucketSpace.getBucketDatabase(), 16, distributorBucketSpace.getDistribution().getRedundancy(), makeBucketSpace()); if (stripToRedundancy) { diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp index 2a220cb9ef8..9838bd9be01 100644 --- a/storage/src/tests/distributor/putoperationtest.cpp +++ b/storage/src/tests/distributor/putoperationtest.cpp @@ -484,7 +484,7 @@ PutOperationTest::getNodes(const std::string& infoString) { std::vector<uint16_t> targetNodes; std::vector<uint16_t> createNodes; - PutOperation::getTargetNodes(getDistributorBucketSpace().get_ideal_nodes(bid), + PutOperation::getTargetNodes(getDistributorBucketSpace().get_ideal_service_layer_nodes_bundle(bid).get_available_nodes(), targetNodes, createNodes, entry, 2); ost << "target( "; diff --git a/storage/src/vespa/storage/config/stor-server.def b/storage/src/vespa/storage/config/stor-server.def index 54160bc53fe..a00f7b4feb5 100644 --- a/storage/src/vespa/storage/config/stor-server.def +++ b/storage/src/vespa/storage/config/stor-server.def @@ -95,5 +95,5 @@ use_content_node_btree_bucket_db bool default=true restart ## WARNING: ## Setting this to a non-zero value requires the minimum split bit level in the cluster ## to be enforced, so only set this value if you know exactly what you're doing! -content_node_bucket_db_stripe_bits int default=0 +content_node_bucket_db_stripe_bits int default=0 restart diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index dd301f0c284..f3ba6af6e0c 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -20,6 +20,7 @@ vespa_add_library(storage_distributor externaloperationhandler.cpp idealstatemanager.cpp idealstatemetricsset.cpp + ideal_service_layer_nodes_bundle.cpp messagetracker.cpp nodeinfo.cpp operation_routing_snapshot.cpp diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp index f9122650311..784ae5ab8af 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp @@ -12,6 +12,8 @@ namespace storage::distributor { namespace { const char *up_states = "uri"; +const char *nonretired_up_states = "ui"; +const char *nonretired_or_maintenance_up_states = "uim"; } @@ -79,12 +81,6 @@ DistributorBucketSpace::setDistribution(std::shared_ptr<const lib::Distribution> clear(); } -std::vector<uint16_t> -DistributorBucketSpace::get_ideal_nodes_fallback(document::BucketId bucket) const -{ - return _distribution->getIdealStorageNodes(*_clusterState, bucket, up_states); -} - void DistributorBucketSpace::set_pending_cluster_state(std::shared_ptr<const lib::ClusterState> pending_cluster_state) { @@ -118,20 +114,41 @@ DistributorBucketSpace::owns_bucket_in_state( return owns_bucket_in_state(*_distribution, clusterState, bucket); } -std::vector<uint16_t> -DistributorBucketSpace::get_ideal_nodes(document::BucketId bucket) const +namespace { + +void +setup_ideal_nodes_bundle(IdealServiceLayerNodesBundle& ideal_nodes_bundle, + const lib::Distribution& distribution, + const lib::ClusterState& cluster_state, + document::BucketId bucket) +{ + ideal_nodes_bundle.set_available_nodes(distribution.getIdealStorageNodes(cluster_state, bucket, up_states)); + ideal_nodes_bundle.set_available_nonretired_nodes(distribution.getIdealStorageNodes(cluster_state, bucket, nonretired_up_states)); + ideal_nodes_bundle.set_available_nonretired_or_maintenance_nodes(distribution.getIdealStorageNodes(cluster_state, bucket, nonretired_or_maintenance_up_states)); +} + +// Ideal service layer nodes bundle used when bucket id used bits > 33. +thread_local IdealServiceLayerNodesBundle fallback_ideal_nodes_bundle; + +} + +const IdealServiceLayerNodesBundle& +DistributorBucketSpace::get_ideal_service_layer_nodes_bundle(document::BucketId bucket) const { assert(bucket.getUsedBits() >= _distribution_bits); - if (bucket.getUsedBits() > 33) { // cf. storage::lib::Distribution::getStorageSeed - // Cannot map to super bucket ==> cannot cache result - return get_ideal_nodes_fallback(bucket); + if (bucket.getUsedBits() > 33) { + IdealServiceLayerNodesBundle &ideal_nodes_bundle = fallback_ideal_nodes_bundle; + setup_ideal_nodes_bundle(ideal_nodes_bundle, *_distribution, *_clusterState, bucket); + return ideal_nodes_bundle; } - document::BucketId super_bucket(_distribution_bits, bucket.getId()); - auto itr = _ideal_nodes.find(super_bucket); + document::BucketId lookup_bucket((bucket.getUsedBits() > 33) ? bucket.getUsedBits() : _distribution_bits, bucket.getId()); + auto itr = _ideal_nodes.find(lookup_bucket); if (itr != _ideal_nodes.end()) { return itr->second; } - auto insres = _ideal_nodes.insert(std::make_pair(super_bucket, get_ideal_nodes_fallback(super_bucket))); + IdealServiceLayerNodesBundle ideal_nodes_bundle; + setup_ideal_nodes_bundle(ideal_nodes_bundle, *_distribution, *_clusterState, lookup_bucket); + auto insres = _ideal_nodes.insert(std::make_pair(lookup_bucket, std::move(ideal_nodes_bundle))); assert(insres.second); return insres.first->second; } diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.h b/storage/src/vespa/storage/distributor/distributor_bucket_space.h index b3722ed9c91..f4d5bd6f5aa 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.h @@ -3,6 +3,7 @@ #include "bucketownership.h" #include "bucket_ownership_flags.h" +#include "ideal_service_layer_nodes_bundle.h" #include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/stllike/hash_map.h> #include <memory> @@ -39,7 +40,7 @@ class DistributorBucketSpace { std::shared_ptr<const lib::ClusterState> _pending_cluster_state; std::vector<bool> _available_nodes; mutable vespalib::hash_map<document::BucketId, BucketOwnershipFlags, document::BucketId::hash> _ownerships; - mutable vespalib::hash_map<document::BucketId, std::vector<uint16_t>, document::BucketId::hash> _ideal_nodes; + mutable vespalib::hash_map<document::BucketId, IdealServiceLayerNodesBundle, document::BucketId::hash> _ideal_nodes; void clear(); void enumerate_available_nodes(); @@ -81,8 +82,6 @@ public: void set_pending_cluster_state(std::shared_ptr<const lib::ClusterState> pending_cluster_state); const lib::ClusterState& get_pending_cluster_state() const noexcept { return *_pending_cluster_state; } - std::vector<uint16_t> get_ideal_nodes_fallback(document::BucketId bucket) const; - /** * Returns true if this distributor owns the given bucket in the * given cluster and current distribution config. @@ -91,10 +90,11 @@ public: bool owns_bucket_in_state(const lib::ClusterState& clusterState, document::BucketId bucket) const; const std::vector<bool>& get_available_nodes() const { return _available_nodes; } + /** - * Returns the ideal nodes for the given bucket. + * Returns the ideal nodes bundle for the given bucket. */ - std::vector<uint16_t> get_ideal_nodes(document::BucketId bucket) const; + const IdealServiceLayerNodesBundle &get_ideal_service_layer_nodes_bundle(document::BucketId bucket) const; /* * Return bucket ownership flags for the given bucket. Bucket is always diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.cpp b/storage/src/vespa/storage/distributor/distributorcomponent.cpp index 86c98cc7b78..d2a2a6fc6a0 100644 --- a/storage/src/vespa/storage/distributor/distributorcomponent.cpp +++ b/storage/src/vespa/storage/distributor/distributorcomponent.cpp @@ -225,7 +225,7 @@ DistributorComponent::updateBucketDatabase( } } - UpdateBucketDatabaseProcessor processor(getClock(), found_down_node ? up_nodes : changedNodes, bucketSpace.get_ideal_nodes(bucket.getBucketId()), (updateFlags & DatabaseUpdate::RESET_TRUSTED) != 0); + UpdateBucketDatabaseProcessor processor(getClock(), found_down_node ? up_nodes : changedNodes, bucketSpace.get_ideal_service_layer_nodes_bundle(bucket.getBucketId()).get_available_nodes(), (updateFlags & DatabaseUpdate::RESET_TRUSTED) != 0); bucketSpace.getBucketDatabase().process_update(bucket.getBucketId(), processor, (updateFlags & DatabaseUpdate::CREATE_IF_NONEXISTING) != 0); } diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp new file mode 100644 index 00000000000..069be02eb10 --- /dev/null +++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp @@ -0,0 +1,17 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "ideal_service_layer_nodes_bundle.h" +#include <vespa/vdslib/distribution/idealnodecalculator.h> + +namespace storage::distributor { + +IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle() noexcept + : _available_nodes(), + _available_nonretired_nodes(), + _available_nonretired_or_maintenance_nodes() +{ +} + +IdealServiceLayerNodesBundle::~IdealServiceLayerNodesBundle() = default; + +} diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h new file mode 100644 index 00000000000..2fd1bc8ad4b --- /dev/null +++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h @@ -0,0 +1,28 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vector> +#include <cstdint> + +namespace storage::distributor { + +/* + * Bundle of ideal service layer nodes for a bucket. + */ +class IdealServiceLayerNodesBundle { + std::vector<uint16_t> _available_nodes; + std::vector<uint16_t> _available_nonretired_nodes; + std::vector<uint16_t> _available_nonretired_or_maintenance_nodes; +public: + IdealServiceLayerNodesBundle() noexcept; + ~IdealServiceLayerNodesBundle(); + + void set_available_nodes(std::vector<uint16_t> available_nodes) { _available_nodes = std::move(available_nodes); } + void set_available_nonretired_nodes(std::vector<uint16_t> available_nonretired_nodes) { _available_nonretired_nodes = std::move(available_nonretired_nodes); } + void set_available_nonretired_or_maintenance_nodes(std::vector<uint16_t> available_nonretired_or_maintenance_nodes) { _available_nonretired_or_maintenance_nodes = std::move(available_nonretired_or_maintenance_nodes); } + std::vector<uint16_t> get_available_nodes() const { return _available_nodes; } + std::vector<uint16_t> get_available_nonretired_nodes() const { return _available_nonretired_nodes; } + std::vector<uint16_t> get_available_nonretired_or_maintenance_nodes() const { return _available_nonretired_or_maintenance_nodes; } +}; + +} diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index 8a4f527c43a..7a247f6c524 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -108,7 +108,7 @@ PutOperation::insertDatabaseEntryAndScheduleCreateBucket(const OperationTargetLi (void) multipleBuckets; BucketDatabase::Entry entry(_bucketSpace.getBucketDatabase().get(lastBucket)); std::vector<uint16_t> idealState( - _bucketSpace.getDistribution().getIdealStorageNodes(_bucketSpace.getClusterState(), lastBucket, "ui")); + _bucketSpace.get_ideal_service_layer_nodes_bundle(lastBucket).get_available_nodes()); active = ActiveCopy::calculate(idealState, _bucketSpace.getDistribution(), entry); LOG(debug, "Active copies for bucket %s: %s", entry.getBucketId().toString().c_str(), active.toString().c_str()); for (uint32_t i=0; i<active.size(); ++i) { @@ -182,10 +182,7 @@ PutOperation::onStart(DistributorMessageSender& sender) if (up) { std::vector<document::BucketId> bucketsToCheckForSplit; - lib::IdealNodeCalculatorImpl idealNodeCalculator; - idealNodeCalculator.setDistribution(_bucketSpace.getDistribution()); - idealNodeCalculator.setClusterState(_bucketSpace.getClusterState()); - OperationTargetResolverImpl targetResolver(_bucketSpace.getBucketDatabase(), idealNodeCalculator, + OperationTargetResolverImpl targetResolver(_bucketSpace, _bucketSpace.getBucketDatabase(), _op_ctx.distributor_config().getMinimalBucketSplit(), _bucketSpace.getDistribution().getRedundancy(), _msg->getBucket().getBucketSpace()); diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp index 23bb6b1db78..7e2da056b5a 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "operationtargetresolverimpl.h" +#include "distributor_bucket_space.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/printable.hpp> #include <sstream> @@ -8,6 +9,20 @@ namespace storage::distributor { +namespace { + +lib::IdealNodeList +make_node_list(const std::vector<uint16_t>& nodes) +{ + lib::IdealNodeList list; + for (auto node : nodes) { + list.push_back(lib::Node(lib::NodeType::STORAGE, node)); + } + return list; +} + +} + BucketInstance::BucketInstance( const document::BucketId& id, const api::BucketInfo& info, lib::Node node, uint16_t idealLocationPriority, bool trusted, bool exist) @@ -54,15 +69,12 @@ BucketInstanceList::add(BucketDatabase::Entry& e, } void -BucketInstanceList::populate(const document::BucketId& specificId, BucketDatabase& db, - const lib::IdealNodeCalculator& idealNodeCalc) +BucketInstanceList::populate(const document::BucketId& specificId, const DistributorBucketSpace& distributor_bucket_space, BucketDatabase& db) { std::vector<BucketDatabase::Entry> entries; db.getParents(specificId, entries); for (uint32_t i=0; i<entries.size(); ++i) { - lib::IdealNodeList idealNodes(idealNodeCalc.getIdealStorageNodes( - entries[i].getBucketId(), - lib::IdealNodeCalculator::UpInitMaintenance)); + lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(entries[i].getBucketId()).get_available_nonretired_or_maintenance_nodes())); add(entries[i], idealNodes); } } @@ -108,17 +120,16 @@ BucketInstanceList::leastSpecificLeafBucketInSubtree( void BucketInstanceList::extendToEnoughCopies( + const DistributorBucketSpace& distributor_bucket_space, const BucketDatabase& db, const document::BucketId& targetIfNonPreExisting, - const document::BucketId& mostSpecificId, - const lib::IdealNodeCalculator& idealNodeCalc) + const document::BucketId& mostSpecificId) { document::BucketId newTarget(_instances.empty() ? targetIfNonPreExisting : _instances[0]._bucket); newTarget = leastSpecificLeafBucketInSubtree(newTarget, mostSpecificId, db); - lib::IdealNodeList idealNodes(idealNodeCalc.getIdealStorageNodes( - newTarget, lib::IdealNodeCalculator::UpInit)); + lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(newTarget).get_available_nonretired_nodes())); for (uint32_t i=0; i<idealNodes.size(); ++i) { if (!contains(idealNodes[i])) { _instances.push_back(BucketInstance( @@ -182,14 +193,14 @@ OperationTargetResolverImpl::getAllInstances(OperationType type, { BucketInstanceList instances; if (type == PUT) { - instances.populate(id, _bucketDatabase, _idealNodeCalculator); + instances.populate(id, _distributor_bucket_space, _bucketDatabase); instances.sort(InstanceOrder()); instances.removeNodeDuplicates(); instances.extendToEnoughCopies( + _distributor_bucket_space, _bucketDatabase, _bucketDatabase.getAppropriateBucket(_minUsedBucketBits, id), - id, - _idealNodeCalculator); + id); } else { throw vespalib::IllegalArgumentException( "Unsupported operation type given", VESPA_STRLOC); diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h index 73a2c281b18..a23c8ba7f59 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h @@ -9,6 +9,8 @@ namespace storage::distributor { +class DistributorBucketSpace; + struct BucketInstance : public vespalib::AsciiPrintable { document::BucketId _bucket; api::BucketInfo _info; @@ -57,13 +59,12 @@ public: * _instances.size() >= configured redundancy level, unless insufficient * number of nodes are available */ - void extendToEnoughCopies(const BucketDatabase& db, + void extendToEnoughCopies(const DistributorBucketSpace& distributor_bucket_space, + const BucketDatabase& db, const document::BucketId& targetIfNonPreExisting, - const document::BucketId& mostSpecificId, - const lib::IdealNodeCalculator& idealNodeCalc); + const document::BucketId& mostSpecificId); - void populate(const document::BucketId&, BucketDatabase&, - const lib::IdealNodeCalculator&); + void populate(const document::BucketId&, const DistributorBucketSpace&, BucketDatabase&); void add(BucketDatabase::Entry& e, const lib::IdealNodeList& idealState); template <typename Order> @@ -77,20 +78,20 @@ public: }; class OperationTargetResolverImpl : public OperationTargetResolver { + const DistributorBucketSpace& _distributor_bucket_space; BucketDatabase& _bucketDatabase; - const lib::IdealNodeCalculator& _idealNodeCalculator; uint32_t _minUsedBucketBits; uint16_t _redundancy; document::BucketSpace _bucketSpace; public: - OperationTargetResolverImpl(BucketDatabase& bucketDatabase, - const lib::IdealNodeCalculator& idealNodeCalc, + OperationTargetResolverImpl(const DistributorBucketSpace& distributor_bucket_space, + BucketDatabase& bucketDatabase, uint32_t minUsedBucketBits, uint16_t redundancy, document::BucketSpace bucketSpace) - : _bucketDatabase(bucketDatabase), - _idealNodeCalculator(idealNodeCalc), + : _distributor_bucket_space(distributor_bucket_space), + _bucketDatabase(bucketDatabase), _minUsedBucketBits(minUsedBucketBits), _redundancy(redundancy), _bucketSpace(bucketSpace) diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java index f8ca8987a70..d614aecbad2 100644 --- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -31,4 +31,9 @@ public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implem peer.start(configFilePath); } + @Override + public boolean reconfigurable() { + return true; + } + } diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index a7f30ab95d8..4dfcbeea444 100644 --- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -38,4 +38,9 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Vespa peer.start(configFilePath); } + @Override + public boolean reconfigurable() { + return false; + } + } diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java index f8ca8987a70..d614aecbad2 100644 --- a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -31,4 +31,9 @@ public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implem peer.start(configFilePath); } + @Override + public boolean reconfigurable() { + return true; + } + } diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index e93bae1e438..2a66bebe048 100644 --- a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -38,4 +38,9 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Vespa peer.start(configFilePath); } + @Override + public boolean reconfigurable() { + return false; + } + } diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java index ba53ea04b6e..c0b5b7c035f 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java @@ -21,4 +21,9 @@ public class DummyVespaZooKeeperServer extends AbstractComponent implements Vesp @Override public void start(Path configFilePath) {} + @Override + public boolean reconfigurable() { + return false; + } + } diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServer.java index ea8c6d2bbee..bb64fcc16ce 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServer.java @@ -6,12 +6,17 @@ import java.nio.file.Path; /** * Interface for a component that starts/stops a ZooKeeper server. * - * @author Harald Musum + * @author hmusum */ public interface VespaZooKeeperServer { + /** Shut down the server. Blocks until shutdown has completed */ void shutdown(); + /** Start the server with the given config file */ void start(Path configFilePath); + /** Whether this server support dynamic reconfiguration */ + boolean reconfigurable(); + } diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java index bf2dd588c93..812c974f9f8 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.zookeeper; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.protect.Process; import com.yahoo.security.tls.TransportSecurityUtils; import java.nio.file.Path; @@ -72,16 +73,26 @@ public class ZooKeeperRunner implements Runnable { startServer(path); // Will block in a real implementation of VespaZooKeeperServer return; } catch (RuntimeException e) { - Duration delay = backoff.delay(attempt); - log.log(Level.WARNING, "Starting ZooKeeper server failed on attempt " + attempt + - ". Retrying in " + delay + ", time left " + Duration.between(now, end), e); - sleeper.sleep(delay); + String messagePart = "Starting " + serverDescription() + " failed on attempt " + + attempt; + if (server.reconfigurable()) { + Duration delay = backoff.delay(attempt); + log.log(Level.WARNING, messagePart + ". Retrying in " + delay + ", time left " + + Duration.between(now, end), e); + sleeper.sleep(delay); + } else { + Process.logAndDie(messagePart + ". Forcing shutdown", e); + } } finally { now = Instant.now(); } } } + private String serverDescription() { + return (server.reconfigurable() ? "" : "non-") + "reconfigurable ZooKeeper server"; + } + private void startServer(Path path) { // Note: Hack to make this work in ZooKeeper 3.6, where metrics provider class is // loaded by using Thread.currentThread().getContextClassLoader() which does not work @@ -91,7 +102,7 @@ public class ZooKeeperRunner implements Runnable { try { server.start(path); } catch (Throwable e) { - throw new RuntimeException("Starting ZooKeeper server failed", e); + throw new RuntimeException("Starting " + serverDescription() + " failed", e); } finally { Thread.currentThread().setContextClassLoader(tccl); } diff --git a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java index 61ddc5996a4..a1b98a23bd0 100644 --- a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java +++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java @@ -165,6 +165,11 @@ public class ReconfigurerTest { @Override public void start(Path configFilePath) { } + @Override + public boolean reconfigurable() { + return true; + } + } private static class TestableVespaZooKeeperAdmin implements VespaZooKeeperAdmin { |