Wire in use-estimate-for-fetch-postings into ExecuteInfo.

10 files changed, 50 insertions, 20 deletions

diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp
index 349d33e03e9..3a2383cfb2f 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp
@@ -99,7 +99,7 @@ AttributeLimiter::create_match_data(size_t want_hits, size_t max_group_size, dou
         FieldSpecList field; // single field API is protected
         field.add(FieldSpec(_attribute_name, my_field_id, my_handle));
         _blueprint = _searchable_attributes.createBlueprint(_requestContext, field, node);
-        auto execInfo = ExecuteInfo::create(strictSearch, strictSearch ? 1.0F : hit_rate, &_requestContext.getDoom(), true);
+        auto execInfo = ExecuteInfo::create(strictSearch, strictSearch ? 1.0F : hit_rate, &_requestContext.getDoom(), true, true);
         _blueprint->fetchPostings(execInfo);
         _estimatedHits.store(_blueprint->getState().estimate().estHits, std::memory_order_relaxed);
         _blueprint->freeze();
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp
index ad1388f0407..7c8608985c3 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp
@@ -17,6 +17,7 @@ using search::queryeval::IDiversifier;
 using search::attribute::diversity::DiversityFilter;
 using search::attribute::BasicType;
 using search::attribute::AttributeBlueprintParams;
+using search::queryeval::ExecuteInfo;
 using vespalib::Issue;
 
 using namespace search::fef::indexproperties::matchphase;
@@ -205,13 +206,15 @@ MatchToolsFactory(QueryLimiter               & queryLimiter,
         trace.addEvent(4, "Perform dictionary lookups and posting lists initialization");
         float hitRate = std::min(1.0F, float(maxNumHits)/float(searchContext.getDocIdLimit()));
         bool create_postinglist_when_non_strict = CreatePostingListWhenNonStrict::check(_queryEnv.getProperties(), rankSetup.create_postinglist_when_non_strict());
-        _query.fetchPostings(search::queryeval::ExecuteInfo::create(is_search, hitRate, &_requestContext.getDoom(),
-                                                                    create_postinglist_when_non_strict));
+        bool use_estimate_for_fetch_postings = UseEstimateForFetchPostings::check(_queryEnv.getProperties(), rankSetup.use_estimate_for_fetch_postings());
+        _query.fetchPostings(ExecuteInfo::create(is_search, hitRate, &_requestContext.getDoom(),
+                                                 create_postinglist_when_non_strict, use_estimate_for_fetch_postings));
         if (is_search) {
             _query.handle_global_filter(_requestContext.getDoom(), searchContext.getDocIdLimit(),
                                         _attribute_blueprint_params.global_filter_lower_limit,
                                         _attribute_blueprint_params.global_filter_upper_limit,
-                                        thread_bundle, trace, create_postinglist_when_non_strict);
+                                        thread_bundle, trace, create_postinglist_when_non_strict,
+                                        use_estimate_for_fetch_postings);
         }
         _query.freeze();
         trace.addEvent(5, "Prepare shared state for multi-threaded rank executors");
diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp
index de0a4846615..d9051ac09e7 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp
@@ -249,7 +249,7 @@ void
 Query::handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit,
                             double global_filter_lower_limit, double global_filter_upper_limit,
                             vespalib::ThreadBundle &thread_bundle, search::engine::Trace& trace,
-                            bool create_postinglist_when_non_strict)
+                            bool create_postinglist_when_non_strict, bool use_estimate_for_fetch_postings)
 {
     if (!handle_global_filter(*_blueprint, docid_limit, global_filter_lower_limit, global_filter_upper_limit, thread_bundle, &trace)) {
         return;
@@ -259,7 +259,7 @@ Query::handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit,
     _blueprint = Blueprint::optimize(std::move(_blueprint));
     LOG(debug, "blueprint after handle_global_filter:\n%s\n", _blueprint->asString().c_str());
     // strictness may change if optimized order changed:
-    fetchPostings(ExecuteInfo::create(true, 1.0F, &doom, create_postinglist_when_non_strict));
+    fetchPostings(ExecuteInfo::create(true, 1.0F, &doom, create_postinglist_when_non_strict, use_estimate_for_fetch_postings));
 }
 
 bool
diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h
index 3e2c7686066..c3ea2c5e664 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/query.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/query.h
@@ -107,7 +107,7 @@ public:
     void handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit,
                               double global_filter_lower_limit, double global_filter_upper_limit,
                               vespalib::ThreadBundle &thread_bundle, search::engine::Trace& trace,
-                              bool create_postinglist_when_non_strict);
+                              bool create_postinglist_when_non_strict, bool use_estimate_for_fetch_postings);
 
     /**
      * Calculates and handles the global filter if needed by the blueprint tree.
diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp
index d503a955afc..637c197b303 100644
--- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp
+++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp
@@ -467,6 +467,12 @@ bool CreatePostingListWhenNonStrict::check(const Properties &props, bool fallbac
     return lookupBool(props, NAME, fallback);
 }
 
+const vespalib::string UseEstimateForFetchPostings::NAME("vespa.matching.use_estimate_for_fetch_postings");
+const bool UseEstimateForFetchPostings::DEFAULT_VALUE(false);
+bool UseEstimateForFetchPostings::check(const Properties &props, bool fallback) {
+    return lookupBool(props, NAME, fallback);
+}
+
 } // namespace matching
 
 namespace softtimeout {
diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h
index 7262e599235..16046709d7e 100644
--- a/searchlib/src/vespa/searchlib/fef/indexproperties.h
+++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h
@@ -357,6 +357,16 @@ namespace matching {
         static bool check(const Properties &props) { return check(props, DEFAULT_VALUE); }
         static bool check(const Properties &props, bool fallback);
     };
+
+    /**
+     * When enabled posting lists can be created on the fly even if iterator is not strict.
+     **/
+    struct UseEstimateForFetchPostings {
+        static const vespalib::string NAME;
+        static const bool DEFAULT_VALUE;
+        static bool check(const Properties &props) { return check(props, DEFAULT_VALUE); }
+        static bool check(const Properties &props, bool fallback);
+    };
 }
 
 namespace softtimeout {
diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
index ed74386b2fa..1f30a3b59e9 100644
--- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
+++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
@@ -62,6 +62,7 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i
       _degradationAscendingOrder(false),
       _always_mark_phrase_expensive(false),
       _create_postinglist_when_non_strict(true),
+      _use_estimate_for_fetch_postings(false),
       _diversityAttribute(),
       _diversityMinGroups(1),
       _diversityCutoffFactor(10.0),
@@ -137,6 +138,7 @@ RankSetup::configure()
     _mutateAllowQueryOverride = mutate::AllowQueryOverride::check(_indexEnv.getProperties());
     _always_mark_phrase_expensive = matching::AlwaysMarkPhraseExpensive::check(_indexEnv.getProperties());
     _create_postinglist_when_non_strict = matching::CreatePostingListWhenNonStrict::check(_indexEnv.getProperties());
+    _use_estimate_for_fetch_postings = matching::UseEstimateForFetchPostings::check(_indexEnv.getProperties());
 }
 
 void
diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h
index d560614ce39..8e4a6c4246e 100644
--- a/searchlib/src/vespa/searchlib/fef/ranksetup.h
+++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h
@@ -71,6 +71,7 @@ private:
     bool                     _degradationAscendingOrder;
     bool                     _always_mark_phrase_expensive;
     bool                     _create_postinglist_when_non_strict;
+    bool                     _use_estimate_for_fetch_postings;
     vespalib::string         _diversityAttribute;
     uint32_t                 _diversityMinGroups;
     double                   _diversityCutoffFactor;
@@ -224,6 +225,7 @@ public:
     }
     bool always_mark_phrase_expensive() const noexcept { return _always_mark_phrase_expensive; }
     bool create_postinglist_when_non_strict() const noexcept { return _create_postinglist_when_non_strict; }
+    bool use_estimate_for_fetch_postings() const noexcept { return _use_estimate_for_fetch_postings; }
     /** get number of hits to collect during graceful degradation in match phase */
     uint32_t getDegradationMaxHits() const {
         return _degradationMaxHits;
diff --git a/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp b/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp
index 27366a9b924..ac3dfcc3b20 100644
--- a/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp
@@ -4,7 +4,7 @@
 
 namespace search::queryeval {
 
-const ExecuteInfo ExecuteInfo::TRUE(true, 1.0, nullptr, true);
-const ExecuteInfo ExecuteInfo::FALSE(false, 1.0, nullptr, true);
+const ExecuteInfo ExecuteInfo::TRUE(true, 1.0, nullptr, true, true);
+const ExecuteInfo ExecuteInfo::FALSE(false, 1.0, nullptr, true, true);
 
 }
diff --git a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h
index 0095429e609..1d6cf7281ea 100644
--- a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h
+++ b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h
@@ -12,43 +12,50 @@ namespace search::queryeval {
  */
 class ExecuteInfo {
 public:
-    ExecuteInfo() noexcept : ExecuteInfo(false, 1.0F, nullptr, true) { }
+    ExecuteInfo() noexcept : ExecuteInfo(false, 1.0F, nullptr, true, true) { }
     bool isStrict() const noexcept { return _strict; }
     bool create_postinglist_when_non_strict() const noexcept { return _create_postinglist_when_non_strict; }
+    bool use_estimate_for_fetch_postings() const noexcept { return _use_estimate_for_fetch_postings; }
     float hitRate() const noexcept { return _hitRate; }
     bool soft_doom() const noexcept { return _doom && _doom->soft_doom(); }
     const vespalib::Doom * getDoom() const { return _doom; }
     static const ExecuteInfo TRUE;
     static const ExecuteInfo FALSE;
     static ExecuteInfo create(bool strict, const ExecuteInfo & org) noexcept {
-        return {strict, org._hitRate, org.getDoom(), org.create_postinglist_when_non_strict()};
+        return create(strict, org._hitRate, org);
     }
     static ExecuteInfo create(bool strict, float hitRate, const ExecuteInfo & org) noexcept {
-        return {strict, hitRate, org.getDoom(), org.create_postinglist_when_non_strict()};
+        return {strict, hitRate, org.getDoom(), org.create_postinglist_when_non_strict(), org.use_estimate_for_fetch_postings()};
     }
-    static ExecuteInfo create(bool strict, float hitRate, const vespalib::Doom * doom) noexcept {
-        return create(strict, hitRate, doom, true);
-    }
-    static ExecuteInfo create(bool strict, float hitRate, const vespalib::Doom * doom, bool postinglist_when_non_strict) noexcept {
-        return {strict, hitRate, doom, postinglist_when_non_strict};
+
+    static ExecuteInfo create(bool strict, float hitRate, const vespalib::Doom * doom, bool postinglist_when_non_strict,
+                              bool use_estimate_for_fetch_postings) noexcept
+    {
+         return {strict, hitRate, doom, postinglist_when_non_strict, use_estimate_for_fetch_postings};
     }
     static ExecuteInfo createForTest(bool strict) noexcept {
         return createForTest(strict, 1.0F);
     }
     static ExecuteInfo createForTest(bool strict, float hitRate) noexcept {
-        return create(strict, hitRate, nullptr);
+        return createForTest(strict, hitRate, nullptr);
+    }
+    static ExecuteInfo createForTest(bool strict, float hitRate, const vespalib::Doom * doom) noexcept {
+        return create(strict, hitRate, doom, true, true);
     }
 private:
-    ExecuteInfo(bool strict, float hitRate_in, const vespalib::Doom * doom, bool postinglist_when_non_strict) noexcept
+    ExecuteInfo(bool strict, float hitRate_in, const vespalib::Doom * doom, bool postinglist_when_non_strict,
+                bool use_estimate_for_fetch_postings) noexcept
         : _doom(doom),
           _hitRate(hitRate_in),
           _strict(strict),
-          _create_postinglist_when_non_strict(postinglist_when_non_strict)
+          _create_postinglist_when_non_strict(postinglist_when_non_strict),
+          _use_estimate_for_fetch_postings(use_estimate_for_fetch_postings)
     { }
     const vespalib::Doom * _doom;
     float                  _hitRate;
     bool                   _strict;
     bool                   _create_postinglist_when_non_strict;
+    bool                   _use_estimate_for_fetch_postings;
 };
 
 }