Merge pull request #5548 from vespa-engine/havardpe/allow-proton-to-drop-sort-data

allow proton to drop sort data when sending query results

6 files changed, 40 insertions, 19 deletions

diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp
index 27e677bb1dc..62c61406f67 100644
--- a/searchcore/src/tests/proton/matching/matching_test.cpp
+++ b/searchcore/src/tests/proton/matching/matching_test.cpp
@@ -20,6 +20,7 @@
 #include <vespa/searchlib/aggregation/perdocexpression.h>
 #include <vespa/searchlib/attribute/extendableattributes.h>
 #include <vespa/searchlib/common/featureset.h>
+#include <vespa/searchlib/common/transport.h>
 #include <vespa/searchlib/engine/docsumrequest.h>
 #include <vespa/searchlib/engine/searchrequest.h>
 #include <vespa/searchlib/engine/docsumreply.h>
@@ -485,20 +486,27 @@ TEST("require that re-ranking is performed (multi-threaded)") {
 }
 
 TEST("require that sortspec can be used (multi-threaded)") {
-    for (size_t threads = 1; threads <= 16; ++threads) {
-        MyWorld world;
-        world.basicSetup();
-        world.basicResults();
-        SearchRequest::SP request = world.createSimpleRequest("f1", "spread");
-        request->sortSpec = "+a1";
-        SearchReply::UP reply = world.performSearch(request, threads);
-        ASSERT_EQUAL(9u, reply->hits.size());
-        EXPECT_EQUAL(document::DocumentId("doc::100").getGlobalId(),  reply->hits[0].gid);
-        EXPECT_EQUAL(zero_rank_value, reply->hits[0].metric);
-        EXPECT_EQUAL(document::DocumentId("doc::200").getGlobalId(),  reply->hits[1].gid);
-        EXPECT_EQUAL(zero_rank_value, reply->hits[1].metric);
-        EXPECT_EQUAL(document::DocumentId("doc::300").getGlobalId(),  reply->hits[2].gid);
-        EXPECT_EQUAL(zero_rank_value, reply->hits[2].metric);
+    for (bool drop_sort_data: {false, true}) {
+        for (size_t threads = 1; threads <= 16; ++threads) {
+            MyWorld world;
+            world.basicSetup();
+            world.basicResults();
+            SearchRequest::SP request = world.createSimpleRequest("f1", "spread");
+            request->sortSpec = "+a1";
+            if (drop_sort_data) {
+                request->queryFlags |= fs4transport::QFLAG_DROP_SORTDATA;
+            }
+            SearchReply::UP reply = world.performSearch(request, threads);
+            ASSERT_EQUAL(9u, reply->hits.size());
+            EXPECT_EQUAL(document::DocumentId("doc::100").getGlobalId(),  reply->hits[0].gid);
+            EXPECT_EQUAL(zero_rank_value, reply->hits[0].metric);
+            EXPECT_EQUAL(document::DocumentId("doc::200").getGlobalId(),  reply->hits[1].gid);
+            EXPECT_EQUAL(zero_rank_value, reply->hits[1].metric);
+            EXPECT_EQUAL(document::DocumentId("doc::300").getGlobalId(),  reply->hits[2].gid);
+            EXPECT_EQUAL(zero_rank_value, reply->hits[2].metric);
+            EXPECT_EQUAL(drop_sort_data, reply->sortIndex.empty());
+            EXPECT_EQUAL(drop_sort_data, reply->sortData.empty());
+        }
     }
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
index 3fbf937cd1b..e285d30b74a 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
@@ -245,7 +245,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl
                            !_rankSetup->getSecondPhaseRank().empty(), !willNotNeedRanking(request, groupingContext));
 
         ResultProcessor rp(attrContext, metaStore, sessionMgr, groupingContext, sessionId,
-                           request.sortSpec, params.offset, params.hits);
+                           request.sortSpec, params.offset, params.hits, request.should_drop_sort_data());
 
         const Properties & rankProperties = request.propertiesMap.rankProperties();
         size_t numThreadsPerSearch = computeNumThreadsPerSearch(mtf->estimate(), rankProperties);
diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
index e2c6affebe1..325803d5aa6 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
@@ -61,7 +61,8 @@ ResultProcessor::ResultProcessor(IAttributeContext &attrContext,
                                  GroupingContext &groupingContext,
                                  const vespalib::string &sessionId,
                                  const vespalib::string &sortSpec,
-                                 size_t offset, size_t hits)
+                                 size_t offset, size_t hits,
+                                 bool drop_sort_data)
     : _attrContext(attrContext),
       _metaStore(metaStore),
       _sessionMgr(sessionMgr),
@@ -70,6 +71,7 @@ ResultProcessor::ResultProcessor(IAttributeContext &attrContext,
       _sortSpec(sortSpec),
       _offset(offset),
       _hits(hits),
+      _drop_sort_data(drop_sort_data),
       _wasMerged(false)
 {
     if (!_groupingContext.empty()) {
@@ -138,7 +140,7 @@ ResultProcessor::makeReply(PartialResultUP full_result)
         dst.metric = src._rankValue;
         LOG(debug, "convertLidToGid: hit[%zu]: lid(%u) -> gid(%s)", i, docId, dst.gid.toString().c_str());
     }
-    if (result.hasSortData() && hitcnt > 0) {
+    if (result.hasSortData() && (hitcnt > 0) && !_drop_sort_data) {
         size_t sortDataSize = result.sortDataSize();
         for (size_t i = 0; i < hitOffset; ++i) {
             sortDataSize -= result.sortData(i).second;
diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h
index a181c1660b7..d32a6986d18 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h
@@ -88,6 +88,7 @@ private:
     const vespalib::string                &_sortSpec;
     size_t                                 _offset;
     size_t                                 _hits;
+    bool                                   _drop_sort_data;
     bool                                   _wasMerged;
 
 public:
@@ -97,7 +98,8 @@ public:
                     GroupingContext & groupingContext,
                     const vespalib::string & sessionId,
                     const vespalib::string & sortSpec,
-                    size_t offset, size_t hits);
+                    size_t offset, size_t hits,
+                    bool drop_sort_data);
     ~ResultProcessor();
 
     size_t countFS4Hits();
diff --git a/searchlib/src/vespa/searchlib/engine/request.cpp b/searchlib/src/vespa/searchlib/engine/request.cpp
index 66b3bbac09b..8f4aa427988 100644
--- a/searchlib/src/vespa/searchlib/engine/request.cpp
+++ b/searchlib/src/vespa/searchlib/engine/request.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "request.h"
+#include <vespa/searchlib/common/transport.h>
 
 namespace search::engine {
 
@@ -16,6 +17,8 @@ Request::Request(const fastos::TimeStamp &start_time)
 {
 }
 
+Request::~Request() = default;
+
 void Request::setTimeout(const fastos::TimeStamp & timeout)
 {
     _timeOfDoom = _startTime + timeout;
@@ -31,6 +34,10 @@ fastos::TimeStamp Request::getTimeLeft() const
     return _timeOfDoom - fastos::TimeStamp(fastos::ClockSystem::now());
 }
 
-Request::~Request() = default;
+bool
+Request::should_drop_sort_data() const
+{
+    return ((queryFlags & fs4transport::QFLAG_DROP_SORTDATA) != 0);
+}
 
 }
diff --git a/searchlib/src/vespa/searchlib/engine/request.h b/searchlib/src/vespa/searchlib/engine/request.h
index d53171e525e..02ab75fe509 100644
--- a/searchlib/src/vespa/searchlib/engine/request.h
+++ b/searchlib/src/vespa/searchlib/engine/request.h
@@ -24,6 +24,8 @@ public:
         return vespalib::stringref(&stackDump[0], stackDump.size());
     }
 
+    bool should_drop_sort_data() const;
+
 private:
     const fastos::TimeStamp _startTime;
     fastos::TimeStamp       _timeOfDoom;