diff options
5 files changed, 25 insertions, 13 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 6fcdd7b0995..41d050025bf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -166,16 +166,17 @@ public class SessionRepository { this.configDefinitionRepo = configDefinitionRepo; this.rewriteSearchDefinitions = Flags.MOVE_SEARCH_DEFINITIONS_TO_SCHEMAS_DIR.bindTo(flagSource); - loadSessions(); // Needs to be done before creating cache below + loadSessions(Flags.LOAD_LOCAL_SESSIONS_WHEN_BOOTSTRAPPING.bindTo(flagSource)); // Needs to be done before creating cache below this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, zkCacheExecutor); this.directoryCache.addListener(this::childEvent); this.directoryCache.start(); } - private void loadSessions() { + private void loadSessions(BooleanFlag loadLocalSessions) { ExecutorService executor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("load-sessions-")); - loadLocalSessions(executor); + if (loadLocalSessions.value()) + loadLocalSessions(executor); loadRemoteSessions(executor); try { executor.shutdown(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index c7270b6c426..10f96ff13cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -340,7 +340,10 @@ public class InternalStepRunner implements StepRunner { .map(since -> since.isBefore(controller.clock().instant().minus(timeouts.noNodesDown()))) .orElse(false)) { if (summary.needPlatformUpgrade() > 0 || summary.needReboot() > 0 || summary.needRestart() > 0) - failureReason = "No nodes allowed to suspend to progress installation for " + timeouts.noNodesDown().toMinutes() + " minutes."; + failureReason = "Timed out after waiting " + timeouts.noNodesDown().toMinutes() + " minutes for " + + "nodes to suspend. This is normal if the cluster is excessively busy. " + + "Nodes will continue to attempt suspension to progress installation independently of " + + "this run."; else failureReason = "Nodes not able to start with new application package."; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 45c423f7353..4885f5c9ae5 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -280,6 +280,12 @@ public class Flags { "Takes effect on next deployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag LOAD_LOCAL_SESSIONS_WHEN_BOOTSTRAPPING = defineFeatureFlag( + "load-local-sessions-when-bootstrapping", true, + List.of("hmusum"), "2021-06-15", "2021-07-15", + "Whether to load local sessions when bootstrapping config server", + "Takes effect on restart of config server"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 448cbc9809d..4e6ae90718c 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -385,19 +385,21 @@ Distributor::random_stripe_idx() } uint32_t -Distributor::stripe_of_bucket_id(const document::BucketId& bucketd_id, api::MessageType::Id msg_id) +Distributor::stripe_of_bucket_id(const document::BucketId& bucket_id, const api::StorageMessage& msg) { - if (!bucketd_id.isSet()) { - // TODO STRIPE: Messages with a non-set bucket id should be handled by the top-level distributor instead. - return 0; - } else if (bucketd_id.getUsedBits() < spi::BucketLimits::MinUsedBits) { - if (msg_id == api::MessageType::VISITOR_CREATE_ID) { + if (!bucket_id.isSet()) { + LOG(error, "Message (%s) has a bucket id (%s) that is not set. Cannot route to stripe", + msg.getSummary().c_str(), bucket_id.toString().c_str()); + } + assert(bucket_id.isSet()); + if (bucket_id.getUsedBits() < spi::BucketLimits::MinUsedBits) { + if (msg.getType().getId() == api::MessageType::VISITOR_CREATE_ID) { // This message will eventually be bounced with api::ReturnCode::WRONG_DISTRIBUTION, // so we can just route it to a random distributor stripe. return random_stripe_idx(); } } - return storage::stripe_of_bucket_key(bucketd_id.toKey(), _n_stripe_bits); + return storage::stripe_of_bucket_key(bucket_id.toKey(), _n_stripe_bits); } bool @@ -413,7 +415,7 @@ Distributor::onDown(const std::shared_ptr<api::StorageMessage>& msg) return true; } auto bucket_id = get_bucket_id_for_striping(*msg, _component); - uint32_t stripe_idx = stripe_of_bucket_id(bucket_id, msg->getType().getId()); + uint32_t stripe_idx = stripe_of_bucket_id(bucket_id, *msg); MBUS_TRACE(msg->getTrace(), 9, vespalib::make_string("Distributor::onDown(): Dispatch message to stripe %u", stripe_idx)); bool handled = _stripes[stripe_idx]->handle_or_enqueue_message(msg); diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index 831750a1c89..6f0808ad63d 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -189,7 +189,7 @@ private: [[nodiscard]] bool may_send_host_info_on_behalf_of_stripes(std::lock_guard<std::mutex>& held_lock) noexcept; uint32_t random_stripe_idx(); - uint32_t stripe_of_bucket_id(const document::BucketId& bucketd_id, api::MessageType::Id msg_id); + uint32_t stripe_of_bucket_id(const document::BucketId& bucket_id, const api::StorageMessage& msg); struct StripeScanStats { bool wants_to_send_host_info = false; |