diff options
author | Martin Polden <mpolden@mpolden.no> | 2020-07-20 16:48:42 +0200 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2020-07-21 10:13:52 +0200 |
commit | aa5768c42fd854c9466baf06d70867bec4531298 (patch) | |
tree | 7afc13388bfa7b9d0a91924895c04ecd124df09f /vespajlib/src/main/java/com | |
parent | bea398a2638d7b1071a2889da771d9fb72ad91d4 (diff) |
Measure consecutive maintenance failures
Measuring time since last success results in a wide range of acceptable values,
due to maintenance intervals varying from seconds to as long as half a day.
Measure consecutive failures instead, to simplify alerting thresholds.
Diffstat (limited to 'vespajlib/src/main/java/com')
-rw-r--r-- | vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java | 24 | ||||
-rw-r--r-- | vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java | 1 |
2 files changed, 13 insertions, 12 deletions
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java index 4c05d46d782..a43e2156025 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java @@ -1,10 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent.maintenance; -import java.time.Clock; -import java.time.Instant; import java.util.Map; -import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.function.BiConsumer; @@ -15,26 +12,29 @@ import java.util.function.BiConsumer; */ public class JobMetrics { - private final Clock clock; - private final BiConsumer<String, Instant> metricConsumer; + private final BiConsumer<String, Long> metricConsumer; - private final Map<String, Instant> successfulRuns = new ConcurrentHashMap<>(); + private final Map<String, Long> incompleteRuns = new ConcurrentHashMap<>(); - public JobMetrics(Clock clock, BiConsumer<String, Instant> metricConsumer) { - this.clock = Objects.requireNonNull(clock); + public JobMetrics(BiConsumer<String, Long> metricConsumer) { this.metricConsumer = metricConsumer; } + /** Record a run for given job */ + public void recordRunOf(String job) { + incompleteRuns.compute(job, (ignored, run) -> run == null ? 1 : ++run); + } + /** Record successful run of given job */ public void recordSuccessOf(String job) { - successfulRuns.put(job, clock.instant()); + incompleteRuns.put(job, 0L); } /** Forward metrics for given job to metric consumer */ public void forward(String job) { - Instant lastSuccess = successfulRuns.get(job); - if (lastSuccess != null) { - metricConsumer.accept(job, lastSuccess); + Long incompleteRuns = this.incompleteRuns.get(job); + if (incompleteRuns != null) { + metricConsumer.accept(job, incompleteRuns); } } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 0385c27536d..eb9b91c812c 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -85,6 +85,7 @@ public abstract class Maintainer implements Runnable, AutoCloseable { public final void lockAndMaintain() { try (var lock = jobControl.lockJob(name())) { try { + jobMetrics.recordRunOf(name()); if (maintain()) jobMetrics.recordSuccessOf(name()); } finally { // Always forward metrics |