diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2017-01-31 23:24:50 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2017-01-31 23:24:50 +0100 |
commit | 398d495c0e5bbde5ca5417ca2ca31e724486c18f (patch) | |
tree | 8bf91f1044708368e73c681c8bfa5f0227cfd282 /searchcore | |
parent | f617123001d2843fed5fa330f0fbc4a1f96f7233 (diff) |
Enable query override for
- vespa.matching.termwise_limit
- vespa.matching.numthreadspersearch
- vespa.matching.numsearchpartitions
- vespa.matching.minhitsperthread
Diffstat (limited to 'searchcore')
5 files changed, 46 insertions, 34 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index fc559930417..db3cf83fd85 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -12,6 +12,7 @@ using search::attribute::IAttributeContext; using search::queryeval::IRequestContext; using namespace search::fef; using namespace search::fef::indexproperties::matchphase; +using namespace search::fef::indexproperties::matching; using search::IDocumentMetaStore; namespace proton { @@ -75,7 +76,8 @@ search::fef::RankProgram::UP MatchTools::first_phase_program() const { auto program = setup_program(_rankSetup.create_first_phase_program(), _mdl, _queryEnv, _featureOverrides); - program->match_data().set_termwise_limit(_rankSetup.get_termwise_limit()); + program->match_data().set_termwise_limit(TermwiseLimit::lookup(_queryEnv.getProperties(), + _rankSetup.get_termwise_limit())); return program; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 15641e7d36d..22bfc8428b0 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -19,6 +19,7 @@ LOG_SETUP(".proton.matching.matcher"); using search::fef::Properties; +using namespace search::fef::indexproperties::matching; using namespace search; using namespace search::engine; using namespace search::grouping; @@ -178,8 +179,7 @@ Matcher::create_match_tools_factory(const search::engine::Request &request, return std::make_unique<MatchToolsFactory>(_queryLimiter, vespalib::Doom(_clock, safeDoom), vespalib::Doom(_clock, request.getTimeOfDoom()), searchContext, attrContext, request.getStackRef(), request.location, _viewResolver, - metaStore, _indexEnv, *_rankSetup, - request.propertiesMap.rankProperties(), feature_overrides); + metaStore, _indexEnv, *_rankSetup, rankProperties, feature_overrides); } SearchReply::UP @@ -196,10 +196,12 @@ Matcher::handleGroupingSession(SessionManager &sessionMgr, return reply; } -size_t Matcher::computeNumThreadsPerSearch(Blueprint::HitEstimate hits) const { - size_t threads = _rankSetup->getNumThreadsPerSearch(); - if ((threads > 1) && (_rankSetup->getMinHitsPerThread() > 0)) { - threads = (hits.empty) ? 1 : std::min(threads, numThreads(hits.estHits, _rankSetup->getMinHitsPerThread())); +size_t +Matcher::computeNumThreadsPerSearch(Blueprint::HitEstimate hits, const Properties & rankProperties) const { + size_t threads = NumThreadsPerSearch::lookup(rankProperties, _rankSetup->getNumThreadsPerSearch()); + uint32_t minHitsPerThread = MinHitsPerThread::lookup(rankProperties, _rankSetup->getMinHitsPerThread()); + if ((threads > 1) && (minHitsPerThread > 0)) { + threads = (hits.empty) ? 1 : std::min(threads, numThreads(hits.estHits, minHitsPerThread)); } return threads; } @@ -219,7 +221,8 @@ Matcher::match(const SearchRequest &request, SearchReply::UP reply = std::make_unique<SearchReply>(); { // we want to measure full set-up and tear-down time as part of // collateral time - GroupingContext groupingContext(_clock, request.getTimeOfDoom(), &request.groupSpec[0], request.groupSpec.size()); + GroupingContext groupingContext(_clock, request.getTimeOfDoom(), + &request.groupSpec[0], request.groupSpec.size()); SessionId sessionId(&request.sessionId[0], request.sessionId.size()); bool shouldCacheSearchSession = false; bool shouldCacheGroupingSession = false; @@ -239,7 +242,8 @@ Matcher::match(const SearchRequest &request, owned_objects.feature_overrides.reset(new Properties(*feature_overrides)); feature_overrides = owned_objects.feature_overrides.get(); } - MatchToolsFactory::UP mtf = create_match_tools_factory(request, searchContext, attrContext, metaStore, *feature_overrides); + MatchToolsFactory::UP mtf = create_match_tools_factory(request, searchContext, attrContext, + metaStore, *feature_overrides); if (!mtf->valid()) { reply->errorCode = ECODE_QUERY_PARSE_ERROR; reply->errorMessage = "query execution failed (invalid query)"; @@ -250,33 +254,42 @@ Matcher::match(const SearchRequest &request, _rankSetup->getRankScoreDropLimit(), request.offset, request.maxhits, !_rankSetup->getSecondPhaseRank().empty(), !willNotNeedRanking(request, groupingContext)); - ResultProcessor rp(attrContext, metaStore, sessionMgr, groupingContext, sessionId, request.sortSpec, params.offset, params.hits); + ResultProcessor rp(attrContext, metaStore, sessionMgr, groupingContext, sessionId, + request.sortSpec, params.offset, params.hits); - size_t numThreadsPerSearch = computeNumThreadsPerSearch(mtf->estimate()); + const Properties & rankProperties = request.propertiesMap.rankProperties(); + size_t numThreadsPerSearch = computeNumThreadsPerSearch(mtf->estimate(), rankProperties); LimitedThreadBundleWrapper limitedThreadBundle(threadBundle, numThreadsPerSearch); MatchMaster master; - ResultProcessor::Result::UP result = master.match(params, limitedThreadBundle, *mtf, rp, _distributionKey, _rankSetup->getNumSearchPartitions()); + uint32_t numSearchPartitions = NumSearchPartitions::lookup(rankProperties, + _rankSetup->getNumSearchPartitions()); + ResultProcessor::Result::UP result = master.match(params, limitedThreadBundle, *mtf, rp, + _distributionKey, numSearchPartitions); my_stats = MatchMaster::getStats(std::move(master)); - size_t estimate = std::min(static_cast<size_t>(metaStore.getCommittedDocIdLimit()), mtf->match_limiter().getDocIdSpaceEstimate()); + size_t estimate = std::min(static_cast<size_t>(metaStore.getCommittedDocIdLimit()), + mtf->match_limiter().getDocIdSpaceEstimate()); bool wasLimited = mtf->match_limiter().was_limited(); if (shouldCacheSearchSession && ((result->_numFs4Hits != 0) || shouldCacheGroupingSession)) { - SearchSession::SP session = std::make_shared<SearchSession>(sessionId, request.getTimeOfDoom(), std::move(mtf), std::move(owned_objects)); + SearchSession::SP session = std::make_shared<SearchSession>(sessionId, request.getTimeOfDoom(), + std::move(mtf), std::move(owned_objects)); session->releaseEnumGuards(); sessionMgr.insert(std::move(session)); } reply = std::move(result->_reply); + SearchReply::Coverage & coverage = reply->coverage; if (wasLimited) { - reply->coverage.degradeMatchPhase(); + coverage.degradeMatchPhase(); } if (my_stats.softDoomed()) { - reply->coverage.degradeTimeout(); + coverage.degradeTimeout(); } - reply->coverage.setActive(metaStore.getNumActiveLids()); - reply->coverage.setSoonActive(metaStore.getNumActiveLids()); //TODO this should be calculated with ClusterState calculator. - reply->coverage.setCovered(std::min(static_cast<size_t>(metaStore.getNumActiveLids()), - (estimate * metaStore.getNumActiveLids())/metaStore.getCommittedDocIdLimit())); + coverage.setActive(metaStore.getNumActiveLids()); + //TODO this should be calculated with ClusterState calculator. + coverage.setSoonActive(metaStore.getNumActiveLids()); + coverage.setCovered(std::min(static_cast<size_t>(metaStore.getNumActiveLids()), + (estimate * metaStore.getNumActiveLids())/metaStore.getCommittedDocIdLimit())); LOG(debug, "numThreadsPerSearch = %d. Configured = %d, estimated hits=%d, totalHits=%ld", - numThreadsPerSearch, _rankSetup->getNumThreadsPerSearch(), mtf->estimate().estHits, reply->totalHitCount); + numThreadsPerSearch, _rankSetup->getNumThreadsPerSearch(), mtf->estimate().estHits, reply->totalHitCount); } total_matching_time.stop(); my_stats.queryCollateralTime(total_matching_time.elapsed().sec() - my_stats.queryLatencyAvg()); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.h b/searchcore/src/vespa/searchcore/proton/matching/matcher.h index d8128f90f9b..88c1116253b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.h @@ -71,7 +71,8 @@ private: search::grouping::GroupingContext & groupingContext, std::unique_ptr<search::grouping::GroupingSession> gs); - size_t computeNumThreadsPerSearch(search::queryeval::Blueprint::HitEstimate hits) const; + size_t computeNumThreadsPerSearch(search::queryeval::Blueprint::HitEstimate hits, + const search::fef::Properties & rankProperties) const; public: /** * Convenience typedefs. diff --git a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h index 490379de7ae..25de6663384 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h +++ b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h @@ -58,24 +58,24 @@ public: } // inherited from search::fef::IQueryEnvironment - virtual const search::fef::Properties &getProperties() const; + const search::fef::Properties &getProperties() const override; // inherited from search::fef::IQueryEnvironment - virtual uint32_t getNumTerms() const; + uint32_t getNumTerms() const override; // inherited from search::fef::IQueryEnvironment - virtual const search::fef::ITermData *getTerm(uint32_t idx) const; + const search::fef::ITermData *getTerm(uint32_t idx) const override; // inherited from search::fef::IQueryEnvironment - virtual const search::fef::Location & getLocation() const; + const search::fef::Location & getLocation() const override; // inherited from search::fef::IQueryEnvironment - virtual const search::attribute::IAttributeContext & getAttributeContext() const; + const search::attribute::IAttributeContext & getAttributeContext() const override; // inherited from search::fef::IQueryEnvironment - virtual const search::fef::IIndexEnvironment & getIndexEnvironment() const; + const search::fef::IIndexEnvironment & getIndexEnvironment() const override; - virtual ~QueryEnvironment(); + ~QueryEnvironment(); }; } // namespace matching diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp index 466db2a7233..e1dec2482a4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp @@ -1,15 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "matchview.h" #include "searchcontext.h" -#include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcore/proton/common/indexsearchabletosearchableadapter.h> -#include <vespa/searchcore/proton/matching/search_session.h> -#include <vespa/searchlib/queryeval/field_spec.h> -#include <vespa/searchlib/queryeval/searchable.h> #include <vespa/searchlib/engine/searchrequest.h> #include <vespa/searchlib/engine/searchreply.h> + #include <vespa/log/log.h> LOG_SETUP(".proton.server.matchview"); |