diff options
author | Lester Solbakken <lesters@oath.com> | 2019-03-04 13:48:57 +0100 |
---|---|---|
committer | Lester Solbakken <lesters@oath.com> | 2019-03-04 13:48:57 +0100 |
commit | dfe9a2b961bbac3747d7cdd7b0fe46c31db5da0f (patch) | |
tree | 2bda2ef3bbbd0be86b06e2569dbf7939556c069c /container-search/src | |
parent | 7a8607f637cb7411c3408ce7499b5f7616b54f2a (diff) |
Revert "Revert "Add relevance @k metric tracking""
This reverts commit 88a5fa4716da6eff95ac941ef50232a81fa25372.
Diffstat (limited to 'container-search/src')
-rw-r--r-- | container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java | 71 |
1 files changed, 69 insertions, 2 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 3606e01ffe5..0b24c400fea 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -15,6 +15,8 @@ import com.yahoo.search.Searcher; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; import com.yahoo.statistics.Counter; @@ -23,6 +25,8 @@ import com.yahoo.statistics.Value; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.PriorityQueue; +import java.util.Queue; import java.util.logging.Level; import static com.yahoo.container.protect.Error.*; @@ -57,6 +61,9 @@ public class StatisticsSearcher extends Searcher { private static final String DOCS_COVERED_METRIC = "documents_covered"; private static final String DOCS_TOTAL_METRIC = "documents_total"; private static final String DEGRADED_METRIC = "degraded_queries"; + private static final String RELEVANCE_AT_1_METRIC = "relevance.at_1"; + private static final String RELEVANCE_AT_3_METRIC = "relevance.at_3"; + private static final String RELEVANCE_AT_10_METRIC = "relevance.at_10"; private final Counter queries; // basic counter private final Counter failedQueries; // basic counter @@ -79,6 +86,7 @@ public class StatisticsSearcher extends Searcher { private Map<String, Metric.Context> chainContexts = new CopyOnWriteHashMap<>(); private Map<String, Metric.Context> statePageOnlyContexts = new CopyOnWriteHashMap<>(); private Map<String, Map<DegradedReason, Metric.Context>> degradedReasonContexts = new CopyOnWriteHashMap<>(); + private Map<String, Map<String, Metric.Context>> relevanceContexts = new CopyOnWriteHashMap<>(); private java.util.Timer scheduler = new java.util.Timer(true); private class PeakQpsReporter extends java.util.TimerTask { @@ -184,6 +192,21 @@ public class StatisticsSearcher extends Searcher { return DegradedReason.non_ideal_state; } + private Metric.Context createRelevanceMetricContext(String chainName, String rankProfile) { + Map<String, String> dimensions = new HashMap<>(); + dimensions.put("chain", chainName); + dimensions.put("rankProfile", rankProfile); + return metric.createContext(dimensions); + } + + private Metric.Context getRelevanceMetricContext(Execution execution, Query query) { + String chain = execution.chain().getId().stringValue(); + String rankProfile = query.getRanking().getProfile(); + return relevanceContexts + .computeIfAbsent(chain, k -> new HashMap<>()) + .computeIfAbsent(rankProfile, k -> createRelevanceMetricContext(chain, rankProfile)); + } + /** * Generate statistics for the query passing through this Searcher * 1) Add 1 to total query count @@ -242,12 +265,12 @@ public class StatisticsSearcher extends Searcher { metric.add(EMPTY_RESULTS_METRIC, 1, metricContext); } - // Update running averages - //setAverages(); + addRelevanceMetrics(query, execution, result); return result; } + private void logQuery(com.yahoo.search.Query query) { // Don't parse the query if it's not necessary for the logging Query.toString triggers parsing if (getLogger().isLoggable(Level.FINER)) { @@ -342,5 +365,49 @@ public class StatisticsSearcher extends Searcher { return context; } + /** + * Effectively flattens the hits, and measures relevance @ 1, 3, and 10 + */ + private void addRelevanceMetrics(Query query, Execution execution, Result result) { + Queue<Double> topScores = findTopRelevanceScores(10, result.hits()); + if (topScores.isEmpty()) { + return; + } + Metric.Context metricContext = getRelevanceMetricContext(execution, query); + setRelevanceMetric(10, RELEVANCE_AT_10_METRIC, topScores, metricContext); // min-queue: lowest values are polled first + setRelevanceMetric(3, RELEVANCE_AT_3_METRIC, topScores, metricContext); + setRelevanceMetric(1, RELEVANCE_AT_1_METRIC, topScores, metricContext); + } + + private static Queue<Double> findTopRelevanceScores(int n, HitGroup hits) { + PriorityQueue<Double> heap = new PriorityQueue<>(n); + for (var iterator = hits.unorderedDeepIterator(); iterator.hasNext(); ) { + Hit hit = iterator.next(); + if (hit instanceof ErrorHit || hit.getRelevance() == null) { + continue; + } + double score = hit.getRelevance().getScore(); + if (Double.isNaN(score)) { + continue; + } + if (heap.size() < n) { + heap.add(score); + } else if (score > heap.peek()) { + heap.remove(); + heap.add(score); + } + } + return heap; + } + + private void setRelevanceMetric(int pos, String name, Queue<Double> minQueue, Metric.Context context) { + while (minQueue.size() > pos) { + minQueue.remove(); + } + if (minQueue.size() == pos) { + metric.set(name, minQueue.poll(), context); + } + } + } |