Merge pull request #27627 from vespa-engine/balder/check-if-document-is-present-when-grouping

Expose bitvector of valid lids and use this when grouping.

11 files changed, 109 insertions, 70 deletions

diff --git a/searchcore/src/tests/grouping/grouping.cpp b/searchcore/src/tests/grouping/grouping.cpp
index a9d3ffe26db..015aec73999 100644
--- a/searchcore/src/tests/grouping/grouping.cpp
+++ b/searchcore/src/tests/grouping/grouping.cpp
@@ -10,6 +10,7 @@
 #include <vespa/searchcore/grouping/groupingmanager.h>
 #include <vespa/searchcore/grouping/groupingsession.h>
 #include <vespa/searchcore/proton/matching/sessionmanager.h>
+#include <vespa/searchlib/common/allocatedbitvector.h>
 #include <vespa/searchlib/test/mock_attribute_context.h>
 #include <vespa/vespalib/util/testclock.h>
 #include <iostream>
@@ -35,8 +36,13 @@ const uint32_t NUM_DOCS = 1000;
 
 struct MyWorld {
     MockAttributeContext attributeContext;
+    search::AllocatedBitVector bv;
 
-    void basicSetup() {
+    MyWorld()
+        : attributeContext(),
+          bv(NUM_DOCS+1)
+    {
+        bv.setInterval(0, NUM_DOCS);
         // attribute context
         {
             SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("attr0");
@@ -170,7 +176,8 @@ TEST_F("testGroupingContextInitialization", DoomFixture()) {
     nos << (uint32_t)1;
     baseRequest.serialize(nos);
 
-    GroupingContext context(f1.clock.clock(), f1.timeOfDoom, os.data(), os.size(), true);
+    AllocatedBitVector bv(1);
+    GroupingContext context(bv, f1.clock.clock(), f1.timeOfDoom, os.data(), os.size(), true);
     ASSERT_TRUE(!context.empty());
     GroupingContext::GroupingList list = context.getGroupingList();
     ASSERT_TRUE(list.size() == 1);
@@ -200,7 +207,8 @@ TEST_F("testGroupingContextUsage", DoomFixture()) {
 
     auto r1 = std::make_shared<Grouping>(request1);
     auto r2 = std::make_shared<Grouping>(request2);
-    GroupingContext context(f1.clock.clock(), f1.timeOfDoom);
+    AllocatedBitVector bv(1);
+    GroupingContext context(bv, f1.clock.clock(), f1.timeOfDoom);
     ASSERT_TRUE(context.empty());
     context.addGrouping(r1);
     ASSERT_TRUE(context.getGroupingList().size() == 1);
@@ -222,7 +230,8 @@ TEST_F("testGroupingContextSerializing", DoomFixture()) {
     nos << (uint32_t)1;
     baseRequest.serialize(nos);
 
-    GroupingContext context(f1.clock.clock(), f1.timeOfDoom);
+    AllocatedBitVector bv(1);
+    GroupingContext context(bv, f1.clock.clock(), f1.timeOfDoom);
     auto bp = std::make_shared<Grouping>(baseRequest);
     context.addGrouping(bp);
     context.serialize();
@@ -240,7 +249,8 @@ TEST_F("testGroupingManager", DoomFixture()) {
             .addLevel(createGL(MU<AttributeNode>("attr1"), MU<AttributeNode>("attr2")))
             .addLevel(createGL(MU<AttributeNode>("attr2"), MU<AttributeNode>("attr3")));
 
-    GroupingContext context(f1.clock.clock(), f1.timeOfDoom);
+    AllocatedBitVector bv(1);
+    GroupingContext context(bv, f1.clock.clock(), f1.timeOfDoom);
     auto bp = std::make_shared<Grouping>(request1);
     context.addGrouping(bp);
     GroupingManager manager(context);
@@ -249,7 +259,6 @@ TEST_F("testGroupingManager", DoomFixture()) {
 
 TEST_F("testGroupingSession", DoomFixture()) {
     MyWorld world;
-    world.basicSetup();
     vespalib::nbostream os;
     Grouping request1;
     request1.setId(0)
@@ -275,7 +284,7 @@ TEST_F("testGroupingSession", DoomFixture()) {
 
     auto r1 = std::make_shared<Grouping>(request1);
     auto r2 = std::make_shared<Grouping>(request2);
-    GroupingContext initContext(f1.clock.clock(), f1.timeOfDoom);
+    GroupingContext initContext(world.bv, f1.clock.clock(), f1.timeOfDoom);
     initContext.addGrouping(r1);
     initContext.addGrouping(r2);
     SessionId id("foo");
@@ -307,7 +316,7 @@ TEST_F("testGroupingSession", DoomFixture()) {
     }
     // Test second pass
     {
-        GroupingContext context(f1.clock.clock(), f1.timeOfDoom);
+        GroupingContext context(world.bv, f1.clock.clock(), f1.timeOfDoom);
         auto r = std::make_shared<Grouping>(request1);
         r->setFirstLevel(1);
         r->setLastLevel(1);
@@ -318,7 +327,7 @@ TEST_F("testGroupingSession", DoomFixture()) {
     }
     // Test last pass. Session should be marked as finished
     {
-        GroupingContext context(f1.clock.clock(), f1.timeOfDoom);
+        GroupingContext context(world.bv, f1.clock.clock(), f1.timeOfDoom);
         auto r = std::make_shared<Grouping>(request1);
         r->setFirstLevel(2);
         r->setLastLevel(2);
@@ -332,7 +341,6 @@ TEST_F("testGroupingSession", DoomFixture()) {
 
 TEST_F("testEmptySessionId", DoomFixture()) {
     MyWorld world;
-    world.basicSetup();
     vespalib::nbostream os;
     Grouping request1;
     request1.setId(0)
@@ -342,7 +350,7 @@ TEST_F("testEmptySessionId", DoomFixture()) {
             .addLevel(createGL(MU<AttributeNode>("attr2"), MU<AttributeNode>("attr3")));
 
     auto r1 = std::make_shared<Grouping>(request1);
-    GroupingContext initContext(f1.clock.clock(), f1.timeOfDoom);
+    GroupingContext initContext(world.bv, f1.clock.clock(), f1.timeOfDoom);
     initContext.addGrouping(r1);
     SessionId id;
 
@@ -362,7 +370,6 @@ TEST_F("testEmptySessionId", DoomFixture()) {
 
 TEST_F("testSessionManager", DoomFixture()) {
     MyWorld world;
-    world.basicSetup();
     vespalib::nbostream os;
     Grouping request1;
     request1.setId(0)
@@ -375,7 +382,7 @@ TEST_F("testSessionManager", DoomFixture()) {
                                                .setResult(Int64ResultNode(0))));
 
     auto r1 = std::make_shared<Grouping>(request1);
-    GroupingContext initContext(f1.clock.clock(), f1.timeOfDoom);
+    GroupingContext initContext(world.bv, f1.clock.clock(), f1.timeOfDoom);
     initContext.addGrouping(r1);
 
     SessionManager mgr(2);
@@ -424,7 +431,6 @@ void doGrouping(GroupingContext &ctx,
 
 TEST_F("test grouping fork/join", DoomFixture()) {
     MyWorld world;
-    world.basicSetup();
 
     Grouping request;
     request.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("attr0"))))
@@ -433,7 +439,7 @@ TEST_F("test grouping fork/join", DoomFixture()) {
            .setLastLevel(1);
 
     auto g1 = std::make_shared<Grouping>(request);
-    GroupingContext context(f1.clock.clock(), f1.timeOfDoom);
+    GroupingContext context(world.bv, f1.clock.clock(), f1.timeOfDoom);
     context.addGrouping(g1);
     GroupingSession session(SessionId(), context, world.attributeContext);
     session.prepareThreadContextCreation(4);
@@ -470,13 +476,12 @@ TEST_F("test grouping fork/join", DoomFixture()) {
 
 TEST_F("test session timeout", DoomFixture()) {
     MyWorld world;
-    world.basicSetup();
     SessionManager mgr(2);
     SessionId id1("foo");
     SessionId id2("bar");
 
-    GroupingContext initContext1(f1.clock.clock(), steady_time(duration(10)));
-    GroupingContext initContext2(f1.clock.clock(), steady_time(duration(20)));
+    GroupingContext initContext1(world.bv, f1.clock.clock(), steady_time(duration(10)));
+    GroupingContext initContext2(world.bv, f1.clock.clock(), steady_time(duration(20)));
     auto s1 = std::make_unique<GroupingSession>(id1, initContext1, world.attributeContext);
     auto s2 = std::make_unique<GroupingSession>(id2, initContext2, world.attributeContext);
     mgr.insert(std::move(s1));
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp
index f76fe873b03..b85dda6a9f2 100644
--- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp
+++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp
@@ -6,6 +6,9 @@
 #include <vespa/searchlib/aggregation/hitsaggregationresult.h>
 #include <vespa/searchlib/common/bitvector.h>
 
+#include <vespa/log/log.h>
+LOG_SETUP(".searchcore/grouping.groupingcontext");
+
 namespace search::grouping {
 
 using aggregation::CountFS4Hits;
@@ -49,9 +52,10 @@ GroupingContext::setDistributionKey(uint32_t distributionKey)
     }
 }
 
-GroupingContext::GroupingContext(const vespalib::Clock & clock, vespalib::steady_time timeOfDoom,
+GroupingContext::GroupingContext(const BitVector & validLids, const vespalib::Clock & clock, vespalib::steady_time timeOfDoom,
                                  const char *groupSpec, uint32_t groupSpecLen, bool enableNested)
-    : _clock(clock),
+    : _validLids(validLids),
+      _clock(clock),
       _timeOfDoom(timeOfDoom),
       _os(),
       _groupingList(),
@@ -60,8 +64,9 @@ GroupingContext::GroupingContext(const vespalib::Clock & clock, vespalib::steady
     deserialize(groupSpec, groupSpecLen);
 }
 
-GroupingContext::GroupingContext(const vespalib::Clock & clock, vespalib::steady_time timeOfDoom)
-    : _clock(clock),
+GroupingContext::GroupingContext(const BitVector & validLids, const vespalib::Clock & clock, vespalib::steady_time timeOfDoom)
+    : _validLids(validLids),
+      _clock(clock),
       _timeOfDoom(timeOfDoom),
       _os(),
       _groupingList(),
@@ -69,7 +74,8 @@ GroupingContext::GroupingContext(const vespalib::Clock & clock, vespalib::steady
 { }
 
 GroupingContext::GroupingContext(const GroupingContext & rhs)
-    : _clock(rhs._clock),
+    : _validLids(rhs._validLids),
+      _clock(rhs._clock),
       _timeOfDoom(rhs._timeOfDoom),
       _os(),
       _groupingList(),
@@ -102,27 +108,32 @@ GroupingContext::needRanking() const
     return true;
 }
 
-using DocId = uint32_t;
+void
+GroupingContext::aggregate(Grouping & grouping, uint32_t docId, HitRank rank) const {
+    if (_validLids.testBit(docId)) {
+        grouping.aggregate(docId, rank);
+    }
+}
 
 unsigned int
 GroupingContext::aggregateRanked(Grouping &grouping, const RankedHit *rankedHit, unsigned int len) const {
     unsigned int i(0);
     for(; (i < len) && !hasExpired(); i++) {
-        grouping.aggregate(rankedHit[i].getDocId(), rankedHit[i].getRank());
+        aggregate(grouping, rankedHit[i].getDocId(), rankedHit[i].getRank());
     }
     return i;
 }
 
 void
 GroupingContext::aggregate(Grouping & grouping, const BitVector * bVec, unsigned int lidLimit) const {
-    for (DocId d(bVec->getFirstTrueBit()); (d < lidLimit) && !hasExpired(); d = bVec->getNextTrueBit(d+1)) {
-        grouping.aggregate(d, 0.0);
+    for (uint32_t d(bVec->getFirstTrueBit()); (d < lidLimit) && !hasExpired(); d = bVec->getNextTrueBit(d+1)) {
+        aggregate(grouping, d, 0.0);
     }
 }
 void
 GroupingContext::aggregate(Grouping & grouping, const BitVector * bVec, unsigned int lidLimit, unsigned int topN) const {
-    for(DocId d(bVec->getFirstTrueBit()), i(0); (d < lidLimit) && (i < topN) && !hasExpired(); d = bVec->getNextTrueBit(d+1), i++) {
-        grouping.aggregate(d, 0.0);
+    for(uint32_t d(bVec->getFirstTrueBit()), i(0); (d < lidLimit) && (i < topN) && !hasExpired(); d = bVec->getNextTrueBit(d+1), i++) {
+        aggregate(grouping, d, 0.0);
     }
 }
 
@@ -153,4 +164,30 @@ GroupingContext::aggregate(Grouping & grouping, const RankedHit * rankedHit, uns
     grouping.postProcess();
 }
 
+void
+GroupingContext::groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow)
+{
+    for (const auto & g : _groupingList) {
+        if ( g->needResort() ) {
+            aggregate(*g, searchResults, binSize, overflow);
+            LOG(debug, "groupUnordered: %s", g->asString().c_str());
+            g->cleanTemporary();
+            g->cleanupAttributeReferences();
+        }
+    }
+}
+
+void
+GroupingContext::groupInRelevanceOrder(const RankedHit *searchResults, uint32_t binSize)
+{
+    for (const auto & g : _groupingList) {
+        if ( ! g->needResort() ) {
+            aggregate(*g, searchResults, binSize);
+            LOG(debug, "groupInRelevanceOrder: %s", g->asString().c_str());
+            g->cleanTemporary();
+            g->cleanupAttributeReferences();
+        }
+    }
+}
+
 }
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h
index 880ffe5f767..6f23fd13378 100644
--- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h
+++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h
@@ -33,7 +33,7 @@ public:
      * @param groupSpec The grouping specification to use for initialization.
      * @param groupSpecLen The length of the grouping specification, in bytes.
      **/
-    GroupingContext(const vespalib::Clock & clock, vespalib::steady_time timeOfDoom,
+    GroupingContext(const BitVector & validLids, const vespalib::Clock & clock, vespalib::steady_time timeOfDoom,
                     const char *groupSpec, uint32_t groupSpecLen, bool enableNestedMultivalueGrouping);
 
     /**
@@ -41,7 +41,7 @@ public:
      * @param groupSpec The grouping specification to use for initialization.
      * @param groupSpecLen The length of the grouping specification, in bytes.
      **/
-    GroupingContext(const vespalib::Clock & clock, vespalib::steady_time timeOfDoom);
+    GroupingContext(const BitVector & validLids, const vespalib::Clock & clock, vespalib::steady_time timeOfDoom);
 
     /**
      * Shallow copy of references
@@ -108,12 +108,16 @@ public:
     bool needRanking() const;
     bool enableNestedMultivalueGrouping() const noexcept { return _enableNestedMultivalueGrouping; }
 
+    void groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow);
+    void groupInRelevanceOrder(const RankedHit *searchResults, uint32_t binSize);
+private:
     void aggregate(Grouping & grouping, const RankedHit * rankedHit, unsigned int len, const BitVector * bv) const;
     void aggregate(Grouping & grouping, const RankedHit * rankedHit, unsigned int len) const;
-private:
+    void aggregate(Grouping & grouping, uint32_t docId, HitRank rank) const;
     unsigned int aggregateRanked(Grouping & grouping, const RankedHit * rankedHit, unsigned int len) const;
     void aggregate(Grouping & grouping, const BitVector * bv, unsigned int lidLimit) const;
     void aggregate(Grouping & grouping, const BitVector * bv, unsigned int , unsigned int topN) const;
+    const BitVector       & _validLids;
     const vespalib::Clock & _clock;
     vespalib::steady_time   _timeOfDoom;
     vespalib::nbostream     _os;
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
index 3eb410ad2c2..e3348f13e18 100644
--- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
+++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
@@ -69,27 +69,13 @@ GroupingManager::init(const IAttributeContext &attrCtx)
 void
 GroupingManager::groupInRelevanceOrder(const RankedHit *searchResults, uint32_t binSize)
 {
-    for (const auto & g : _groupingContext.getGroupingList()) {
-        if ( ! g->needResort() ) {
-            _groupingContext.aggregate(*g, searchResults, binSize);
-            LOG(debug, "groupInRelevanceOrder: %s", g->asString().c_str());
-            g->cleanTemporary();
-            g->cleanupAttributeReferences();
-        }
-    }
+    _groupingContext.groupInRelevanceOrder(searchResults, binSize);
 }
 
 void
 GroupingManager::groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow)
 {
-    for (const auto & g : _groupingContext.getGroupingList()) {
-        if ( g->needResort() ) {
-            _groupingContext.aggregate(*g, searchResults, binSize, overflow);
-            LOG(debug, "groupUnordered: %s", g->asString().c_str());
-            g->cleanTemporary();
-            g->cleanupAttributeReferences();
-        }
-    }
+    _groupingContext.groupUnordered(searchResults, binSize, overflow);
 }
 
 void
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
index f0d345b4b5a..d8f42968147 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
@@ -202,6 +202,7 @@ public:
     DocId getNumActiveLids() const override { return _lidAlloc.getNumActiveLids(); }
     search::LidUsageStats getLidUsageStats() const override;
     std::unique_ptr<search::queryeval::Blueprint> createWhiteListBlueprint() const override;
+    const search::BitVector & getValidLids() const override { return _lidAlloc.getUsedLids(); }
 
     /**
      * Implements search::AttributeVector
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h
index 67f5f01ba60..6b4d8844263 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h
@@ -85,6 +85,7 @@ public:
     }
 
     const search::BitVector &getActiveLids() const { return _activeLids.getBitVector(); }
+    const search::BitVector &getUsedLids() const { return _usedLids.getBitVector(); }
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
index acbd069aac7..9ce8fcd9269 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
@@ -328,9 +328,7 @@ MatchThread::findMatches(MatchTools &tools)
 }
 
 void
-MatchThread::processResult(const Doom & doom,
-                           search::ResultSet::UP result,
-                           ResultProcessor::Context &context)
+MatchThread::processResult(const Doom & doom, search::ResultSet::UP result, ResultProcessor::Context &context)
 {
     if (doom.hard_doom()) return;
     bool hasGrouping = bool(context.grouping);
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h
index 607a9d6aab0..db30ea8d2b2 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h
@@ -120,18 +120,18 @@ private:
     using RankSetup = search::fef::RankSetup;
     using IIndexEnvironment = search::fef::IIndexEnvironment;
     using IDiversifier = search::queryeval::IDiversifier;
-    QueryLimiter              & _queryLimiter;
-    AttributeBlueprintParams    _global_filter_params;
-    Query                       _query;
-    MaybeMatchPhaseLimiter::UP  _match_limiter;
+    QueryLimiter                     & _queryLimiter;
+    AttributeBlueprintParams           _global_filter_params;
+    Query                              _query;
+    MaybeMatchPhaseLimiter::UP         _match_limiter;
     std::unique_ptr<RangeQueryLocator> _rangeLocator;
-    QueryEnvironment            _queryEnv;
-    RequestContext              _requestContext;
-    MatchDataLayout             _mdl;
-    const RankSetup           & _rankSetup;
-    const Properties          & _featureOverrides;
-    DiversityParams             _diversityParams;
-    bool                        _valid;
+    QueryEnvironment                   _queryEnv;
+    RequestContext                     _requestContext;
+    MatchDataLayout                    _mdl;
+    const RankSetup                  & _rankSetup;
+    const Properties                 & _featureOverrides;
+    DiversityParams                    _diversityParams;
+    bool                               _valid;
 
     std::unique_ptr<AttributeOperationTask>
     createTask(vespalib::stringref attribute, vespalib::stringref operation) const;
@@ -172,7 +172,7 @@ public:
     std::unique_ptr<AttributeOperationTask> createOnSummaryTask() const;
 
     const Query & query() const { return _query; }
-    const RequestContext & getRequestContext() const { return _requestContext; }
+    const RequestContext & get_request_context() const { return _requestContext; }
 
     const StringStringMap & get_feature_rename_map() const;
 
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
index 26cf9901f57..aedfde2521c 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
@@ -16,6 +16,7 @@
 #include <vespa/searchlib/fef/indexproperties.h>
 #include <vespa/searchlib/fef/ranksetup.h>
 #include <vespa/searchlib/fef/test/plugin/setup.h>
+#include <vespa/searchlib/common/allocatedbitvector.h>
 #include <vespa/vespalib/data/slime/inserter.h>
 #include <cinttypes>
 
@@ -51,6 +52,8 @@ constexpr vespalib::duration TIME_BEFORE_ALLOWING_SOFT_TIMEOUT_FACTOR_ADJUSTMENT
 
 // used to give out empty whitelist blueprints
 struct StupidMetaStore : search::IDocumentMetaStore {
+    static const search::AllocatedBitVector _dummy;
+    const search::BitVector & getValidLids() const override { return _dummy; }
     bool getGid(DocId, GlobalId &) const override { return false; }
     bool getGidEvenIfMoved(DocId, GlobalId &) const override { return false; }
     bool getLid(const GlobalId &, DocId &) const override { return false; }
@@ -65,6 +68,8 @@ struct StupidMetaStore : search::IDocumentMetaStore {
     void foreach(const search::IGidToLidMapperVisitor &) const override { }
 };
 
+const search::AllocatedBitVector StupidMetaStore::_dummy(1);
+
 size_t
 numThreads(size_t hits, size_t minHits) {
     return static_cast<size_t>(std::ceil(double(hits) / double(minHits)));
@@ -245,7 +250,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl
     bool isDoomExplicit = false;
     { // we want to measure full set-up and tear-down time as part of
       // collateral time
-        GroupingContext groupingContext(_clock, request.getTimeOfDoom(),
+        GroupingContext groupingContext(metaStore.getValidLids(), _clock, request.getTimeOfDoom(),
                                         request.groupSpec.data(), request.groupSpec.size(), _rankSetup->enableNestedMultivalueGrouping());
         SessionId sessionId(request.sessionId.data(), request.sessionId.size());
         bool shouldCacheSearchSession = false;
@@ -270,7 +275,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl
 
         MatchToolsFactory::UP mtf = create_match_tools_factory(request, searchContext, attrContext, metaStore,
                                                                *feature_overrides, threadBundle, &owned_objects.readGuard, true);
-        isDoomExplicit = mtf->getRequestContext().getDoom().isExplicitSoftDoom();
+        isDoomExplicit = mtf->get_request_context().getDoom().isExplicitSoftDoom();
         traceQuery(6, request.trace(), mtf->query());
         if (!mtf->valid()) {
             return reply;
diff --git a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h
index b21cc809794..89ed8312ec4 100644
--- a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h
+++ b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h
@@ -51,6 +51,9 @@ struct DocumentMetaStoreObserver : public IDocumentMetaStore
     uint64_t getCurrentGeneration() const override {
         return _store.getCurrentGeneration();
     }
+    const search::BitVector & getValidLids() const override {
+        return _store.getValidLids();
+    }
 
 
     /**
diff --git a/searchlib/src/vespa/searchlib/common/idocumentmetastore.h b/searchlib/src/vespa/searchlib/common/idocumentmetastore.h
index 66b4905ea64..729bede1e71 100644
--- a/searchlib/src/vespa/searchlib/common/idocumentmetastore.h
+++ b/searchlib/src/vespa/searchlib/common/idocumentmetastore.h
@@ -55,13 +55,10 @@ struct DocumentMetaData {
     }
 };
 
-namespace queryeval {
-
-class Blueprint;
-
-}
+namespace queryeval { class Blueprint; }
 
 class IGidToLidMapperVisitor;
+class BitVector;
 
 
 /**
@@ -77,6 +74,8 @@ struct IDocumentMetaStore {
 
     virtual ~IDocumentMetaStore() = default;
 
+    virtual const BitVector & getValidLids() const = 0;
+
     /**
      * Retrieves the gid associated with the given lid.
      * Returns true if found, false otherwise.