From bed93841e3ccb7ef4d8fa084d4edd8c0b43b4c06 Mon Sep 17 00:00:00 2001
From: Geir Storli <geirst@yahooinc.com>
Date: Thu, 20 Oct 2022 15:38:53 +0000
Subject: Return the full search iterator when all lids are active.

This opens up for optimizations of the global filter iterator tree
based on SearchIterator::matches_any().
---
 .../lid_allocator/lid_allocator_test.cpp           | 37 ++++++++++++++++++++++
 .../proton/documentmetastore/lid_allocator.cpp     | 13 ++++++--
 2 files changed, 48 insertions(+), 2 deletions(-)

(limited to 'searchcore')

diff --git a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp
index 8d8674da4f0..19212160496 100644
--- a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp
+++ b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp
@@ -1,13 +1,18 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/searchcore/proton/documentmetastore/lid_allocator.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/simpleresult.h>
 #include <vespa/vespalib/util/generationholder.h>
 #include <vespa/vespalib/util/time.h>
 #include <vespa/vespalib/gtest/gtest.h>
 #include <iostream>
 
+using search::queryeval::Blueprint;
+using search::queryeval::SimpleResult;
 using vespalib::GenerationHolder;
 using vespalib::Timer;
+using vespalib::Trinary;
 
 namespace proton {
 
@@ -94,6 +99,22 @@ protected:
         return result;
     }
 
+    SimpleResult get_active_lids_in_search_iterator(uint32_t docid_limit) {
+        auto blueprint = _allocator.createWhiteListBlueprint();
+        blueprint->setDocIdLimit(docid_limit);
+        auto iterator = blueprint->createFilterSearch(true, search::queryeval::Blueprint::FilterConstraint::UPPER_BOUND);
+        SimpleResult res;
+        res.search(*iterator, docid_limit);
+        return res;
+    }
+
+    Trinary search_iterator_matches_any(uint32_t docid_limit) {
+        auto blueprint = _allocator.createWhiteListBlueprint();
+        blueprint->setDocIdLimit(docid_limit);
+        auto iterator = blueprint->createFilterSearch(true, search::queryeval::Blueprint::FilterConstraint::UPPER_BOUND);
+        return iterator->matches_any();
+    }
+
     void
     assert_valid_lids(const std::vector<uint32_t>& exp_lids) {
         EXPECT_EQ(exp_lids, get_valid_lids());
@@ -121,6 +142,22 @@ TEST_F(LidAllocatorTest, unregister_lids)
     EXPECT_EQ((std::vector<uint32_t>{1, 3, 5, 7, 8}), alloc_lids(5));
 }
 
+TEST_F(LidAllocatorTest, active_lids_are_available_in_search_iterator)
+{
+    register_lids({ 1, 2, 3, 4 });
+    activate_lids({ 1, 2, 4 }, true);
+    EXPECT_EQ(Trinary::Undefined, search_iterator_matches_any(5));
+    EXPECT_EQ(SimpleResult({1, 2, 4}), get_active_lids_in_search_iterator(5));
+}
+
+TEST_F(LidAllocatorTest, search_iterator_matches_all_when_all_lids_are_active)
+{
+    register_lids({ 1, 2, 3, 4 });
+    activate_lids({ 1, 2, 3, 4 }, true);
+    EXPECT_EQ(Trinary::True, search_iterator_matches_any(5));
+    EXPECT_EQ(SimpleResult({1, 2, 3, 4}), get_active_lids_in_search_iterator(5));
+}
+
 class LidAllocatorPerformanceTest : public LidAllocatorTest,
                                     public testing::WithParamInterface<bool>
 {
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp
index 57f8be576db..1d02ed4f063 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp
@@ -4,6 +4,7 @@
 #include <vespa/searchlib/common/bitvectoriterator.h>
 #include <vespa/searchlib/fef/termfieldmatchdataarray.h>
 #include <vespa/searchlib/fef/matchdata.h>
+#include <vespa/searchlib/queryeval/full_search.h>
 #include <mutex>
 
 #include <vespa/log/log.h>
@@ -12,6 +13,7 @@ LOG_SETUP(".proton.documentmetastore.lid_allocator");
 using search::fef::TermFieldMatchDataArray;
 using search::queryeval::Blueprint;
 using search::queryeval::FieldSpecBaseList;
+using search::queryeval::FullSearch;
 using search::queryeval::SearchIterator;
 using search::queryeval::SimpleLeafBlueprint;
 using vespalib::GenerationHolder;
@@ -182,6 +184,7 @@ class WhiteListBlueprint : public SimpleLeafBlueprint
 {
 private:
     const search::BitVector &_activeLids;
+    bool _all_lids_active;
     mutable std::mutex _lock;
     mutable std::vector<search::fef::TermFieldMatchData *> _matchDataVector;
 
@@ -193,9 +196,11 @@ private:
         return createFilterSearch(strict, FilterConstraint::UPPER_BOUND);
     }
 public:
-    WhiteListBlueprint(const search::BitVector &activeLids)
+    WhiteListBlueprint(const search::BitVector &activeLids, bool all_lids_active)
         : SimpleLeafBlueprint(FieldSpecBaseList()),
           _activeLids(activeLids),
+          _all_lids_active(all_lids_active),
+          _lock(),
           _matchDataVector()
     {
         setEstimate(HitEstimate(_activeLids.size(), false));
@@ -204,6 +209,9 @@ public:
     bool isWhiteList() const override { return true; }
 
     SearchIterator::UP createFilterSearch(bool strict, FilterConstraint) const override {
+        if (_all_lids_active) {
+            return std::make_unique<FullSearch>();
+        }
         auto tfmd = new search::fef::TermFieldMatchData;
         {
             std::lock_guard<std::mutex> lock(_lock);
@@ -224,7 +232,8 @@ public:
 Blueprint::UP
 LidAllocator::createWhiteListBlueprint() const
 {
-    return std::make_unique<WhiteListBlueprint>(_activeLids.getBitVector());
+    return std::make_unique<WhiteListBlueprint>(_activeLids.getBitVector(),
+                                                (getNumUsedLids() == getNumActiveLids()));
 }
 
 void
-- 
cgit v1.2.3