diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2016-07-06 17:11:58 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2016-07-06 17:11:58 +0000 |
commit | 78e7a81c74fa452b2ee636ca3913d0130c3e5939 (patch) | |
tree | 8f67b12655050e61c2fa45dc3d21c68728528d9a /searchlib/src | |
parent | 5816b7cf411fef217d9add56595ec2dbf04ea4bb (diff) |
In order to reduce overhead we look at the hit estimate before fanning out to multiple threads.
Default is as today, configurable per rank-profile via min-hits-per-thread.
Default should be changed to a sane number once verified.
The main intentions is to counter the sometimes high cost of per thread rank-setup.
Diffstat (limited to 'searchlib/src')
7 files changed, 36 insertions, 2 deletions
diff --git a/searchlib/src/tests/fef/properties/properties_test.cpp b/searchlib/src/tests/fef/properties/properties_test.cpp index a08d511b418..a904fd13e49 100644 --- a/searchlib/src/tests/fef/properties/properties_test.cpp +++ b/searchlib/src/tests/fef/properties/properties_test.cpp @@ -267,6 +267,15 @@ TEST("test stuff") { p.add("vespa.matching.numthreadspersearch", "50"); EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); } + + { // vespa.matching.minhitsperthread + EXPECT_EQUAL(matching::MinHitsPerThread::NAME, vespalib::string("vespa.matching.minhitsperthread")); + EXPECT_EQUAL(matching::MinHitsPerThread::DEFAULT_VALUE, 0); + Properties p; + EXPECT_EQUAL(matching::MinHitsPerThread::lookup(p), 0); + p.add("vespa.matching.minhitsperthread", "50"); + EXPECT_EQUAL(matching::MinHitsPerThread::lookup(p), 50u); + } { EXPECT_EQUAL(matching::NumSearchPartitions::NAME, vespalib::string("vespa.matching.numsearchpartitions")); EXPECT_EQUAL(matching::NumSearchPartitions::DEFAULT_VALUE, 1u); diff --git a/searchlib/src/tests/ranksetup/ranksetup_test.cpp b/searchlib/src/tests/ranksetup/ranksetup_test.cpp index aee04ef4cb7..ead2f09766d 100644 --- a/searchlib/src/tests/ranksetup/ranksetup_test.cpp +++ b/searchlib/src/tests/ranksetup/ranksetup_test.cpp @@ -485,6 +485,7 @@ void RankSetupTest::testRankSetup() env.getProperties().add(dump::Feature::NAME, "foo"); env.getProperties().add(dump::Feature::NAME, "bar"); env.getProperties().add(matching::NumThreadsPerSearch::NAME, "3"); + env.getProperties().add(matching::MinHitsPerThread::NAME, "8"); env.getProperties().add(matchphase::DegradationAttribute::NAME, "mystaticrankattr"); env.getProperties().add(matchphase::DegradationAscendingOrder::NAME, "true"); env.getProperties().add(matchphase::DegradationMaxHits::NAME, "12345"); @@ -509,6 +510,7 @@ void RankSetupTest::testRankSetup() EXPECT_EQUAL(rs.getDumpFeatures()[0], vespalib::string("foo")); EXPECT_EQUAL(rs.getDumpFeatures()[1], vespalib::string("bar")); EXPECT_EQUAL(rs.getNumThreadsPerSearch(), 3u); + EXPECT_EQUAL(rs.getMinHitsPerThread(), 8u); EXPECT_EQUAL(rs.getDegradationAttribute(), "mystaticrankattr"); EXPECT_EQUAL(rs.isDegradationOrderAscending(), true); EXPECT_EQUAL(rs.getDegradationMaxHits(), 12345u); diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp index 14d92e3a909..40d2b0ba811 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp @@ -161,6 +161,17 @@ NumSearchPartitions::lookup(const Properties &props) { return lookupUint32(props, NAME, DEFAULT_VALUE); } + +const vespalib::string MinHitsPerThread::NAME("vespa.matching.minhitsperthread"); +const uint32_t MinHitsPerThread::DEFAULT_VALUE(0); + +uint32_t +MinHitsPerThread::lookup(const Properties &props) +{ + return lookupUint32(props, NAME, DEFAULT_VALUE); +} + + } // namespace matching namespace matchphase { diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h index 8dcd08dfc49..626206836ca 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.h +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h @@ -105,6 +105,14 @@ namespace matching { /** * Property for the number of threads used per search. **/ + struct MinHitsPerThread { + static const vespalib::string NAME; + static const uint32_t DEFAULT_VALUE; + static uint32_t lookup(const Properties &props); + }; + /** + * Property for the number of threads used per search. + **/ struct NumSearchPartitions { static const vespalib::string NAME; static const uint32_t DEFAULT_VALUE; diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index a954f70c82b..bfc79482080 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -36,6 +36,7 @@ RankSetup::RankSetup(const BlueprintFactory &factory, _secondPhaseRankFeature(), _degradationAttribute(), _numThreads(0), + _minHitsPerThread(0), _numSearchPartitions(0), _heapSize(0), _arraySize(0), @@ -75,6 +76,7 @@ RankSetup::configure() } set_termwise_limit(indexproperties::matching::TermwiseLimit::lookup(_indexEnv.getProperties())); setNumThreadsPerSearch(indexproperties::matching::NumThreadsPerSearch::lookup(_indexEnv.getProperties())); + setMinHitsPerThread(indexproperties::matching::MinHitsPerThread::lookup(_indexEnv.getProperties())); setNumSearchPartitions(indexproperties::matching::NumSearchPartitions::lookup(_indexEnv.getProperties())); setHeapSize(indexproperties::hitcollector::HeapSize::lookup(_indexEnv.getProperties())); setArraySize(indexproperties::hitcollector::ArraySize::lookup(_indexEnv.getProperties())); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index 86b381e3af6..d5acb13e117 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -34,6 +34,7 @@ private: vespalib::string _degradationAttribute; double _termwise_limit; uint32_t _numThreads; + uint32_t _minHitsPerThread; uint32_t _numSearchPartitions; uint32_t _heapSize; uint32_t _arraySize; @@ -146,6 +147,8 @@ public: * @return the number of threads **/ uint32_t getNumThreadsPerSearch() const { return _numThreads; } + uint32_t getMinHitsPerThread() const { return _minHitsPerThread; } + void setMinHitsPerThread(uint32_t minHitsPerThread) { _minHitsPerThread = minHitsPerThread; } void setNumSearchPartitions(uint32_t numSearchPartitions) { _numSearchPartitions = numSearchPartitions; } diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index 2700f218321..b01d60d529c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -259,8 +259,7 @@ public: Blueprint::UP removeChild(size_t n); SearchIterator::UP createSearch(fef::MatchData &md, bool strict) const override; - virtual HitEstimate - combine(const std::vector<HitEstimate> &data) const = 0; + virtual HitEstimate combine(const std::vector<HitEstimate> &data) const = 0; virtual FieldSpecBaseList exposeFields() const = 0; virtual void sort(std::vector<Blueprint*> &children) const = 0; virtual bool inheritStrict(size_t i) const = 0; |