diff options
author | Lester Solbakken <lesters@users.noreply.github.com> | 2019-03-12 11:26:52 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-03-12 11:26:52 +0100 |
commit | 0cb431662aa0fe0ed6094cb4fc60c28b84e6e3bb (patch) | |
tree | f6af117be3c2780f145dd67b35856324dec2b2ad | |
parent | a4cab61c260b1ef166d592ede3b51dcbfbb13445 (diff) | |
parent | 0f6cdc41db8d0ad676a104875271320223da6698 (diff) |
Merge pull request #8677 from vespa-engine/lesters/add-relevance-tracking-2
Lesters/add relevance tracking 2
-rw-r--r-- | config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java | 7 | ||||
-rw-r--r-- | container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java | 79 |
2 files changed, 84 insertions, 2 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 7881ef70ebb..5e7c0d9a7a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -219,6 +219,13 @@ public class VespaMetricSet { metrics.add(new Metric("requestsOverQuota.rate")); metrics.add(new Metric("requestsOverQuota.count")); + metrics.add(new Metric("relevance.at_1.average")); + metrics.add(new Metric("relevance.at_1.count")); + metrics.add(new Metric("relevance.at_3.average")); + metrics.add(new Metric("relevance.at_3.count")); + metrics.add(new Metric("relevance.at_10.average")); + metrics.add(new Metric("relevance.at_10.count")); + // Errors from qrserver metrics.add(new Metric("error.timeout.rate")); metrics.add(new Metric("error.backends_oos.rate")); diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 3606e01ffe5..086ceb514fa 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -15,6 +15,8 @@ import com.yahoo.search.Searcher; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; import com.yahoo.statistics.Counter; @@ -23,6 +25,8 @@ import com.yahoo.statistics.Value; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.PriorityQueue; +import java.util.Queue; import java.util.logging.Level; import static com.yahoo.container.protect.Error.*; @@ -57,6 +61,9 @@ public class StatisticsSearcher extends Searcher { private static final String DOCS_COVERED_METRIC = "documents_covered"; private static final String DOCS_TOTAL_METRIC = "documents_total"; private static final String DEGRADED_METRIC = "degraded_queries"; + private static final String RELEVANCE_AT_1_METRIC = "relevance.at_1"; + private static final String RELEVANCE_AT_3_METRIC = "relevance.at_3"; + private static final String RELEVANCE_AT_10_METRIC = "relevance.at_10"; private final Counter queries; // basic counter private final Counter failedQueries; // basic counter @@ -79,6 +86,7 @@ public class StatisticsSearcher extends Searcher { private Map<String, Metric.Context> chainContexts = new CopyOnWriteHashMap<>(); private Map<String, Metric.Context> statePageOnlyContexts = new CopyOnWriteHashMap<>(); private Map<String, Map<DegradedReason, Metric.Context>> degradedReasonContexts = new CopyOnWriteHashMap<>(); + private Map<String, Map<String, Metric.Context>> relevanceContexts = new CopyOnWriteHashMap<>(); private java.util.Timer scheduler = new java.util.Timer(true); private class PeakQpsReporter extends java.util.TimerTask { @@ -184,6 +192,29 @@ public class StatisticsSearcher extends Searcher { return DegradedReason.non_ideal_state; } + private Metric.Context createRelevanceMetricContext(String chainName, String rankProfile) { + Map<String, String> dimensions = new HashMap<>(); + dimensions.put("chain", chainName); + dimensions.put("rankProfile", rankProfile); + return metric.createContext(dimensions); + } + + private Metric.Context getRelevanceMetricContext(Execution execution, Query query) { + String chain = execution.chain().getId().stringValue(); + String rankProfile = query.getRanking().getProfile(); + Map<String, Metric.Context> chainContext = relevanceContexts.get(chain); // CopyOnWriteHashMap - don't use computeIfAbsent + if (chainContext == null) { + chainContext = new CopyOnWriteHashMap<>(); + relevanceContexts.put(chain, chainContext); + } + Metric.Context metricContext = chainContext.get(rankProfile); + if (metricContext == null) { + metricContext = createRelevanceMetricContext(chain, rankProfile); + chainContext.put(rankProfile, metricContext); + } + return metricContext; + } + /** * Generate statistics for the query passing through this Searcher * 1) Add 1 to total query count @@ -242,12 +273,12 @@ public class StatisticsSearcher extends Searcher { metric.add(EMPTY_RESULTS_METRIC, 1, metricContext); } - // Update running averages - //setAverages(); + addRelevanceMetrics(query, execution, result); return result; } + private void logQuery(com.yahoo.search.Query query) { // Don't parse the query if it's not necessary for the logging Query.toString triggers parsing if (getLogger().isLoggable(Level.FINER)) { @@ -342,5 +373,49 @@ public class StatisticsSearcher extends Searcher { return context; } + /** + * Effectively flattens the hits, and measures relevance @ 1, 3, and 10 + */ + private void addRelevanceMetrics(Query query, Execution execution, Result result) { + Queue<Double> topScores = findTopRelevanceScores(10, result.hits()); + if (topScores.isEmpty()) { + return; + } + Metric.Context metricContext = getRelevanceMetricContext(execution, query); + setRelevanceMetric(10, RELEVANCE_AT_10_METRIC, topScores, metricContext); // min-queue: lowest values are polled first + setRelevanceMetric(3, RELEVANCE_AT_3_METRIC, topScores, metricContext); + setRelevanceMetric(1, RELEVANCE_AT_1_METRIC, topScores, metricContext); + } + + private static Queue<Double> findTopRelevanceScores(int n, HitGroup hits) { + PriorityQueue<Double> heap = new PriorityQueue<>(n); + for (var iterator = hits.unorderedDeepIterator(); iterator.hasNext(); ) { + Hit hit = iterator.next(); + if (hit instanceof ErrorHit || hit.getRelevance() == null) { + continue; + } + double score = hit.getRelevance().getScore(); + if (Double.isNaN(score)) { + continue; + } + if (heap.size() < n) { + heap.add(score); + } else if (score > heap.peek()) { + heap.remove(); + heap.add(score); + } + } + return heap; + } + + private void setRelevanceMetric(int pos, String name, Queue<Double> minQueue, Metric.Context context) { + while (minQueue.size() > pos) { + minQueue.remove(); + } + if (minQueue.size() == pos) { + metric.set(name, minQueue.poll(), context); + } + } + } |