Revert "Revert "Balder/rearrange threads""

3 files changed, 18 insertions, 6 deletions

diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def
index 8f5b22aa7fa..33b3ec37c04 100644
--- a/storage/src/vespa/storage/config/stor-communicationmanager.def
+++ b/storage/src/vespa/storage/config/stor-communicationmanager.def
@@ -29,11 +29,11 @@ mbus.compress.type enum {NONE, LZ4, ZSTD} default=LZ4
 ## TTL for rpc target cache
 mbus.rpctargetcache.ttl double default = 600
 
-## Number of threads for mbus threadpool
+## Number of threads for network.
 ## Any value below 1 will be 1.
-mbus.num_threads int default=4
+mbus.num_threads int default=1
 
-mbus.optimize_for enum {LATENCY, THROUGHPUT} default = LATENCY
+mbus.optimize_for enum {LATENCY, THROUGHPUT} default = THROUGHPUT
 
 ## Enable to use above thread pool for encoding replies
 ## False will use network(fnet) thread
@@ -42,4 +42,4 @@ mbus.dispatch_on_encode bool default=true
 ## Enable to use above thread pool for decoding replies
 ## False will use network(fnet) thread
 ## Todo: Change default once verified in large scale deployment.
-mbus.dispatch_on_decode bool default=false
+mbus.dispatch_on_decode bool default=true
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
index fa2b0cda018..e7d1f06bbd7 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
@@ -25,6 +25,7 @@ LOG_SETUP(".communication.manager");
 
 using vespalib::make_string;
 using document::FixedBucketSpaces;
+using CommunicationManagerConfig = vespa::config::content::core::StorCommunicationmanagerConfig;
 
 namespace storage {
 
@@ -281,6 +282,17 @@ struct PlaceHolderBucketResolver : public BucketResolver {
     }
 };
 
+mbus::RPCNetworkParams::OptimizeFor
+convert(CommunicationManagerConfig::Mbus::OptimizeFor optimizeFor) {
+    switch (optimizeFor) {
+        case CommunicationManagerConfig::Mbus::OptimizeFor::LATENCY:
+            return mbus::RPCNetworkParams::OptimizeFor::LATENCY;
+        case CommunicationManagerConfig::Mbus::OptimizeFor::THROUGHPUT:
+        default:
+            return mbus::RPCNetworkParams::OptimizeFor::THROUGHPUT;
+    }
+}
+
 }
 
 CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, const config::ConfigUri & configUri)
@@ -415,7 +427,7 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig>
         params.setNumThreads(std::max(1, config->mbus.numThreads));
         params.setDispatchOnDecode(config->mbus.dispatchOnDecode);
         params.setDispatchOnEncode(config->mbus.dispatchOnEncode);
-        params.setTcpNoDelay(config->mbus.optimizeFor == CommunicationManagerConfig::Mbus::OptimizeFor::LATENCY);
+        params.setOptimizeFor(convert(config->mbus.optimizeFor));
 
         params.setIdentity(mbus::Identity(_component.getIdentity()));
         if (config->mbusport != -1) {
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h
index c08ad214768..a0ae4bf3b43 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.h
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.h
@@ -116,7 +116,7 @@ private:
 
     void process(const std::shared_ptr<api::StorageMessage>& msg);
 
-    using CommunicationManagerConfig= vespa::config::content::core::StorCommunicationmanagerConfig;
+    using CommunicationManagerConfig = vespa::config::content::core::StorCommunicationmanagerConfig;
     using BucketspacesConfig = vespa::config::content::core::BucketspacesConfig;
 
     void configureMessageBusLimits(const CommunicationManagerConfig& cfg);