Compute number of covered documents if degraded by soft timeout.

6 files changed, 39 insertions, 15 deletions

diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp
index 46cba2d59c4..b94026e689c 100644
--- a/searchcore/src/tests/proton/matching/matching_stats_test.cpp
+++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp
@@ -1,13 +1,16 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("matching_stats_test");
+
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/searchcore/proton/matching/matching_stats.h>
 
+#include <vespa/log/log.h>
+LOG_SETUP("matching_stats_test");
+
 using namespace proton::matching;
 
 TEST("requireThatDocCountsAddUp") {
     MatchingStats stats;
+    EXPECT_EQUAL(0u, stats.docsCovered());
     EXPECT_EQUAL(0u, stats.docsMatched());
     EXPECT_EQUAL(0u, stats.docsRanked());
     EXPECT_EQUAL(0u, stats.docsReRanked());
@@ -15,6 +18,7 @@ TEST("requireThatDocCountsAddUp") {
     EXPECT_EQUAL(0u, stats.limited_queries());
     {
         MatchingStats rhs;
+        EXPECT_EQUAL(&rhs.docsCovered(10000), &rhs);
         EXPECT_EQUAL(&rhs.docsMatched(1000), &rhs);
         EXPECT_EQUAL(&rhs.docsRanked(100), &rhs);
         EXPECT_EQUAL(&rhs.docsReRanked(10), &rhs);
@@ -22,13 +26,15 @@ TEST("requireThatDocCountsAddUp") {
         EXPECT_EQUAL(&rhs.limited_queries(1), &rhs);
         EXPECT_EQUAL(&stats.add(rhs), &stats);
     }
+    EXPECT_EQUAL(10000u, stats.docsCovered());
     EXPECT_EQUAL(1000u, stats.docsMatched());
     EXPECT_EQUAL(100u, stats.docsRanked());
     EXPECT_EQUAL(10u, stats.docsReRanked());
     EXPECT_EQUAL(2u, stats.queries());
     EXPECT_EQUAL(1u, stats.limited_queries());
-    EXPECT_EQUAL(&stats.add(MatchingStats().docsMatched(1000).docsRanked(100)
+    EXPECT_EQUAL(&stats.add(MatchingStats().docsCovered(10000).docsMatched(1000).docsRanked(100)
                             .docsReRanked(10).queries(2).limited_queries(1)), &stats);
+    EXPECT_EQUAL(20000u, stats.docsCovered());
     EXPECT_EQUAL(2000u, stats.docsMatched());
     EXPECT_EQUAL(200u, stats.docsRanked());
     EXPECT_EQUAL(20u, stats.docsReRanked());
@@ -84,12 +90,14 @@ TEST("requireThatAverageTimesAreRecorded") {
 
 TEST("requireThatPartitionsAreAddedCorrectly") {
     MatchingStats all1;
+    EXPECT_EQUAL(0u, all1.docsCovered());
     EXPECT_EQUAL(0u, all1.docsMatched());
     EXPECT_EQUAL(0u, all1.getNumPartitions());
 
     MatchingStats::Partition subPart;
-    subPart.docsMatched(3).docsRanked(2).docsReRanked(1)
+    subPart.docsCovered(7).docsMatched(3).docsRanked(2).docsReRanked(1)
         .active_time(1.0).wait_time(0.5);
+    EXPECT_EQUAL(7u, subPart.docsCovered());
     EXPECT_EQUAL(3u, subPart.docsMatched());
     EXPECT_EQUAL(2u, subPart.docsRanked());
     EXPECT_EQUAL(1u, subPart.docsReRanked());
@@ -99,10 +107,12 @@ TEST("requireThatPartitionsAreAddedCorrectly") {
     EXPECT_EQUAL(1u, subPart.wait_time_count());
 
     all1.merge_partition(subPart, 0);
+    EXPECT_EQUAL(7u, all1.docsCovered());
     EXPECT_EQUAL(3u, all1.docsMatched());
     EXPECT_EQUAL(2u, all1.docsRanked());
     EXPECT_EQUAL(1u, all1.docsReRanked());
     EXPECT_EQUAL(1u, all1.getNumPartitions());
+    EXPECT_EQUAL(7u, all1.getPartition(0).docsCovered());
     EXPECT_EQUAL(3u, all1.getPartition(0).docsMatched());
     EXPECT_EQUAL(2u, all1.getPartition(0).docsRanked());
     EXPECT_EQUAL(1u, all1.getPartition(0).docsReRanked());
@@ -112,6 +122,7 @@ TEST("requireThatPartitionsAreAddedCorrectly") {
     EXPECT_EQUAL(1u, all1.getPartition(0).wait_time_count());
     
     all1.merge_partition(subPart, 1);
+    EXPECT_EQUAL(14u, all1.docsCovered());
     EXPECT_EQUAL(6u, all1.docsMatched());
     EXPECT_EQUAL(4u, all1.docsRanked());
     EXPECT_EQUAL(2u, all1.docsReRanked());
@@ -125,6 +136,7 @@ TEST("requireThatPartitionsAreAddedCorrectly") {
     EXPECT_EQUAL(1u, all1.getPartition(1).wait_time_count());
 
     all1.add(all1);
+    EXPECT_EQUAL(28u, all1.docsCovered());
     EXPECT_EQUAL(12u, all1.docsMatched());
     EXPECT_EQUAL(8u, all1.docsRanked());
     EXPECT_EQUAL(4u, all1.docsReRanked());
diff --git a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h
index 67e52cbe8ee..e6fce89e82d 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h
@@ -10,8 +10,7 @@
 #include <algorithm>
 #include <vector>
 
-namespace proton {
-namespace matching {
+namespace proton::matching {
 
 /**
  * A range of document ids representing a subset of the search space.
@@ -205,5 +204,4 @@ public:
     DocidRange share_range(size_t, DocidRange todo) override;
 };
 
-} // namespace proton::matching
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
index bd6a06ac4f1..9483e119c60 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
@@ -140,7 +140,7 @@ MatchThread::try_share(DocidRange &docid_range, uint32_t next_docid) {
 }
 
 template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work>
-bool
+uint32_t
 MatchThread::inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range)
 {
     SearchIterator *search = &tools.search();
@@ -164,7 +164,7 @@ MatchThread::inner_match_loop(Context &context, MatchTools &tools, DocidRange do
             docId = Strategy::seek_next(*search, docId + 1);
         }
     }
-    return (docId < docid_range.end);
+    return docId;
 }
 
 template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work>
@@ -172,12 +172,15 @@ void
 MatchThread::match_loop(MatchTools &tools, HitCollector &hits)
 {
     bool softDoomed = false;
+    uint32_t covered = 0;
     Context context(matchParams.rankDropLimit, tools, hits, num_threads);
     for (DocidRange docid_range = scheduler.first_range(thread_id);
          !docid_range.empty() && ! softDoomed;
          docid_range = scheduler.next_range(thread_id))
     {
-        softDoomed = inner_match_loop<Strategy, do_rank, do_limit, do_share_work>(context, tools, docid_range);
+        uint32_t lastCovered = inner_match_loop<Strategy, do_rank, do_limit, do_share_work>(context, tools, docid_range);
+        softDoomed = (lastCovered < docid_range.end);
+        covered += lastCovered - docid_range.begin;
     }
     uint32_t matches = context.matches;
     if (do_limit && context.isBelowLimit()) {
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
index b08323fa099..b7ecf149001 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
@@ -75,7 +75,7 @@ private:
     bool try_share(DocidRange &docid_range, uint32_t next_docid) __attribute__((noinline));
 
     template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work>
-    bool inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) __attribute__((noinline));
+    uint32_t inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) __attribute__((noinline));
 
     template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work>
     void match_loop(MatchTools &tools, HitCollector &hits) __attribute__((noinline));
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
index de46386d6e1..7ea417bb7d2 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
@@ -18,6 +18,7 @@ MatchingStats::Partition &get_writable_partition(std::vector<MatchingStats::Part
 MatchingStats::MatchingStats()
     : _queries(0),
       _limited_queries(0),
+      _docsCovered(0),
       _docsMatched(0),
       _docsRanked(0),
       _docsReRanked(0),
@@ -31,13 +32,14 @@ MatchingStats::MatchingStats()
       _partitions()
 { }
 
-MatchingStats::~MatchingStats() { }
+MatchingStats::~MatchingStats() = default;
 
 MatchingStats &
 MatchingStats::merge_partition(const Partition &partition, size_t id)
 {
     get_writable_partition(_partitions, id) = partition;
 
+    _docsCovered += partition.docsCovered();
     _docsMatched += partition.docsMatched();
     _docsRanked += partition.docsRanked();
     _docsReRanked += partition.docsReRanked();
@@ -51,10 +53,10 @@ MatchingStats::merge_partition(const Partition &partition, size_t id)
 MatchingStats &
 MatchingStats::add(const MatchingStats &rhs)
 {
-
     _queries += rhs._queries;
     _limited_queries += rhs._limited_queries;
 
+    _docsCovered += rhs._docsCovered;
     _docsMatched += rhs._docsMatched;
     _docsRanked += rhs._docsRanked;
     _docsReRanked += rhs._docsReRanked;
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h
index 99084098f6f..e4578547e60 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h
@@ -37,6 +37,7 @@ public:
      * thread.
      **/
     class Partition {
+        size_t _docsCovered;
         size_t _docsMatched;
         size_t _docsRanked;
         size_t _docsReRanked;
@@ -45,13 +46,16 @@ public:
         Avg    _wait_time;
     public:
         Partition()
-            : _docsMatched(0),
+            : _docsCovered(0),
+              _docsMatched(0),
               _docsRanked(0),
               _docsReRanked(0),
               _softDoomed(0),
               _active_time(),
               _wait_time() { }
 
+        Partition &docsCovered(size_t value) { _docsCovered = value; return *this; }
+        size_t docsCovered() const { return _docsCovered; }
         Partition &docsMatched(size_t value) { _docsMatched = value; return *this; }
         size_t docsMatched() const { return _docsMatched; }
         Partition &docsRanked(size_t value) { _docsRanked = value; return *this; }
@@ -69,6 +73,7 @@ public:
         size_t wait_time_count() const { return _wait_time.count(); }
 
         Partition &add(const Partition &rhs) {
+            _docsCovered += rhs.docsCovered();
             _docsMatched += rhs._docsMatched;
             _docsRanked += rhs._docsRanked;
             _docsReRanked += rhs._docsReRanked;
@@ -83,6 +88,7 @@ public:
 private:
     size_t                 _queries;
     size_t                 _limited_queries;
+    size_t                 _docsCovered;
     size_t                 _docsMatched;
     size_t                 _docsRanked;
     size_t                 _docsReRanked;
@@ -109,6 +115,9 @@ public:
     MatchingStats &limited_queries(size_t value) { _limited_queries = value; return *this; }
     size_t limited_queries() const { return _limited_queries; }
 
+    MatchingStats &docsCovered(size_t value) { _docsCovered = value; return *this; }
+    size_t docsCovered() const { return _docsCovered; }
+
     MatchingStats &docsMatched(size_t value) { _docsMatched = value; return *this; }
     size_t docsMatched() const { return _docsMatched; }