Add configurable support for weakly consistent client Gets

If configured, Get operations initiated by the client are flagged
with weak internal consistency. This allows the backend to bypass
certain internal synchronization mechanisms, which minimizes latency
at the cost of possibly not observing a consistent view of the
document. This config should only be used in a very restricted set
of cases where the document set is effectively read-only, or cross-
field consistency or freshness does not matter.

To enable the weak consistency, use an explicit config override:

```
<config name="vespa.config.content.core.stor-distributormanager">
  <use_weak_internal_read_consistency_for_client_gets>
    true
  </use_weak_internal_read_consistency_for_client_gets>
</config>
```

This closes #11811

4 files changed, 86 insertions, 2 deletions

diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp
index 83ef7891630..afc3b254c64 100644
--- a/storage/src/tests/distributor/distributortest.cpp
+++ b/storage/src/tests/distributor/distributortest.cpp
@@ -187,6 +187,12 @@ struct DistributorTest : Test, DistributorTestUtil {
         configureDistributor(builder);
     }
 
+    void configure_use_weak_internal_read_consistency(bool use_weak) {
+        ConfigBuilder builder;
+        builder.useWeakInternalReadConsistencyForClientGets = use_weak;
+        configureDistributor(builder);
+    }
+
     void configureMaxClusterClockSkew(int seconds);
     void sendDownClusterStateCommand();
     void replyToSingleRequestBucketInfoCommandWith1Bucket();
@@ -1038,6 +1044,19 @@ TEST_F(DistributorTest, merge_disabling_config_is_propagated_to_internal_config)
     EXPECT_FALSE(getConfig().merge_operations_disabled());
 }
 
+TEST_F(DistributorTest, weak_internal_read_consistency_config_is_propagated_to_internal_configs) {
+    createLinks(true);
+    setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
+
+    configure_use_weak_internal_read_consistency(true);
+    EXPECT_TRUE(getConfig().use_weak_internal_read_consistency_for_client_gets());
+    EXPECT_TRUE(getExternalOperationHandler().use_weak_internal_read_consistency_for_gets());
+
+    configure_use_weak_internal_read_consistency(false);
+    EXPECT_FALSE(getConfig().use_weak_internal_read_consistency_for_client_gets());
+    EXPECT_FALSE(getExternalOperationHandler().use_weak_internal_read_consistency_for_gets());
+}
+
 TEST_F(DistributorTest, concurrent_reads_not_enabled_if_btree_db_is_not_enabled) {
     createLinks(false);
     setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
diff --git a/storage/src/tests/distributor/externaloperationhandlertest.cpp b/storage/src/tests/distributor/externaloperationhandlertest.cpp
index f2f949a6d56..03ba148277e 100644
--- a/storage/src/tests/distributor/externaloperationhandlertest.cpp
+++ b/storage/src/tests/distributor/externaloperationhandlertest.cpp
@@ -83,6 +83,7 @@ struct ExternalOperationHandlerTest : Test, DistributorTestUtil {
         close();
     }
 
+    void do_test_get_weak_consistency_is_propagated(bool use_weak);
 };
 
 TEST_F(ExternalOperationHandlerTest, bucket_split_mask) {
@@ -534,6 +535,32 @@ TEST_F(ExternalOperationHandlerTest, gets_are_busy_bounced_during_transition_per
               _sender.reply(0)->getResult().toString());
 }
 
+void ExternalOperationHandlerTest::do_test_get_weak_consistency_is_propagated(bool use_weak) {
+    createLinks();
+    setupDistributor(1, 2, "version:1 distributor:1 storage:1");
+    // Explicitly only touch config in the case weak consistency is enabled to ensure the
+    // default is strong.
+    if (use_weak) {
+        getExternalOperationHandler().set_use_weak_internal_read_consistency_for_gets(true);
+    }
+    document::BucketId b(16, 1234);
+    Operation::SP op;
+    ASSERT_NO_FATAL_FAILURE(start_operation_verify_not_rejected(
+            makeGetCommandForUser(b.withoutCountBits()), op));
+    auto& get_op = dynamic_cast<GetOperation&>(*op);
+    EXPECT_EQ(get_op.desired_read_consistency(),
+              (use_weak ? api::InternalReadConsistency::Weak
+                        : api::InternalReadConsistency::Strong));
+}
+
+TEST_F(ExternalOperationHandlerTest, gets_are_sent_with_strong_consistency_by_default) {
+    do_test_get_weak_consistency_is_propagated(false);
+}
+
+TEST_F(ExternalOperationHandlerTest, gets_are_sent_with_weak_consistency_if_config_enabled) {
+    do_test_get_weak_consistency_is_propagated(true);
+}
+
 // TODO support sequencing of RemoveLocation? It's a mutating operation, but supporting it with
 // the current approach is not trivial. A RemoveLocation operation covers the _entire_ bucket
 // sub tree under a given location, while the sequencer works on individual GIDs. Mapping the
diff --git a/storage/src/tests/distributor/getoperationtest.cpp b/storage/src/tests/distributor/getoperationtest.cpp
index db790639869..0dbec8444cc 100644
--- a/storage/src/tests/distributor/getoperationtest.cpp
+++ b/storage/src/tests/distributor/getoperationtest.cpp
@@ -51,12 +51,13 @@ struct GetOperationTest : Test, DistributorTestUtil {
         op.reset();
     }
 
-    void sendGet() {
+    void sendGet(api::InternalReadConsistency consistency = api::InternalReadConsistency::Strong) {
         auto msg = std::make_shared<api::GetCommand>(makeDocumentBucket(document::BucketId(0)), docId, "[all]");
         op = std::make_unique<GetOperation>(
                 getExternalOperationHandler(), getDistributorBucketSpace(),
                 getDistributorBucketSpace().getBucketDatabase().acquire_read_guard(),
-                msg, getDistributor().getMetrics(). gets[msg->getLoadType()]);
+                msg, getDistributor().getMetrics(). gets[msg->getLoadType()],
+                consistency);
         op->start(_sender, framework::MilliSecTime(0));
     }
 
@@ -127,6 +128,8 @@ struct GetOperationTest : Test, DistributorTestUtil {
     void setClusterState(const std::string& clusterState) {
         enableDistributorClusterState(clusterState);
     }
+
+    void do_test_read_consistency_is_propagated(api::InternalReadConsistency consistency);
 };
 
 GetOperationTest::GetOperationTest() = default;
@@ -497,4 +500,23 @@ TEST_F(GetOperationTest, can_get_documents_when_all_replica_nodes_retired) {
     EXPECT_EQ("Get => 0", _sender.getCommands(true));
 }
 
+void GetOperationTest::do_test_read_consistency_is_propagated(api::InternalReadConsistency consistency) {
+    setClusterState("distributor:1 storage:1");
+    addNodesToBucketDB(bucketId, "0=4");
+    sendGet(consistency);
+    ASSERT_TRUE(op);
+    EXPECT_EQ(dynamic_cast<GetOperation&>(*op).desired_read_consistency(), consistency);
+    ASSERT_EQ("Get => 0", _sender.getCommands(true));
+    auto& cmd = dynamic_cast<const api::GetCommand&>(*_sender.command(0));
+    EXPECT_EQ(cmd.internal_read_consistency(), consistency);
+}
+
+TEST_F(GetOperationTest, can_send_gets_with_strong_internal_read_consistency) {
+    do_test_read_consistency_is_propagated(api::InternalReadConsistency::Strong);
+}
+
+TEST_F(GetOperationTest, can_send_gets_with_weak_internal_read_consistency) {
+    do_test_read_consistency_is_propagated(api::InternalReadConsistency::Weak);
+}
+
 }
diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp
index c8bfcd754f7..f1900c40d56 100644
--- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp
+++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp
@@ -1088,6 +1088,22 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_document_not_foun
     ASSERT_EQ("Put => 1,Put => 0", sender.getCommands(true, false, 2));
 }
 
+// The weak consistency config _only_ applies to Get operations initiated directly
+// by the client, not those indirectly initiated by the distributor in order to
+// fulfill update write-repairs.
+TEST_F(TwoPhaseUpdateOperationTest, update_gets_are_sent_with_strong_consistency_even_if_weak_consistency_configured) {
+    setupDistributor(2, 2, "storage:2 distributor:1");
+    getConfig().set_use_weak_internal_read_consistency_for_client_gets(true);
+
+    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=2/3/4")); // Inconsistent replicas.
+    DistributorMessageSenderStub sender;
+    cb->start(sender, framework::MilliSecTime(0));
+
+    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
+    auto& get_cmd = dynamic_cast<const api::GetCommand&>(*sender.command(0));
+    EXPECT_EQ(get_cmd.internal_read_consistency(), api::InternalReadConsistency::Strong);
+}
+
 // XXX currently differs in behavior from content nodes in that updates for
 // document IDs without explicit doctypes will _not_ be auto-failed on the
 // distributor.