Revert "Revert "Emit a success factor from maintainers""

This reverts commit cd1b747b4f65fa3a6ed6aace23235db7591638c5.

5 files changed, 63 insertions, 33 deletions

diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java
index fcc5b8e57a2..73a5dc77743 100644
--- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java
+++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java
@@ -2,25 +2,18 @@
 package com.yahoo.concurrent.maintenance;
 
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.BiConsumer;
 
 /**
  * Tracks and forwards maintenance job metrics.
  *
  * @author mpolden
  */
-public class JobMetrics {
-
-    private final BiConsumer<String, Long> metricConsumer;
+public abstract class JobMetrics {
 
     private final ConcurrentHashMap<String, Long> incompleteRuns = new ConcurrentHashMap<>();
 
-    public JobMetrics(BiConsumer<String, Long> metricConsumer) {
-        this.metricConsumer = metricConsumer;
-    }
-
-    /** Record a run for given job */
-    public void recordRunOf(String job) {
+    /** Record starting of a run of a job */
+    public void starting(String job) {
         incompleteRuns.merge(job, 1L, Long::sum);
     }
 
@@ -29,12 +22,17 @@ public class JobMetrics {
         incompleteRuns.put(job, 0L);
     }
 
-    /** Forward metrics for given job to metric consumer */
-    public void forward(String job) {
+    /**
+     * Records completion of a run of a job.
+     * This is guaranteed to always be called once whenever starting has been called.
+     */
+    public void completed(String job, double successFactor) {
         Long incompleteRuns = this.incompleteRuns.get(job);
         if (incompleteRuns != null) {
-            metricConsumer.accept(job, incompleteRuns);
+            recordCompletion(job, incompleteRuns, successFactor);
         }
     }
 
+    protected abstract void recordCompletion(String job, Long incompleteRuns, double successFactor);
+
 }
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
index 734c46a2819..2a9e6dda6b6 100644
--- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
+++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
@@ -83,8 +83,19 @@ public abstract class Maintainer implements Runnable {
     @Override
     public final String toString() { return name(); }
 
-    /** Called once each time this maintenance job should run. Returns whether the maintenance run was successful */
-    protected abstract boolean maintain();
+    /**
+     * Called once each time this maintenance job should run.
+     *
+     * @return the degree to which the run was successful - a number between 0 (no success), to 1 (complete success).
+     *         Note that this indicates whether something is wrong, so e.g if the call did nothing because it should do
+     *         nothing,  1.0 should be returned.
+     */
+    protected abstract double maintain();
+
+    /** Convenience methods to convert attempts and failures into a success factor */
+    protected final double asSuccessFactor(int attempts, int failures) {
+        return attempts == 0 ? 1.0 : 1 - (double)failures / attempts;
+    }
 
     /** Returns the interval at which this job is set to run */
     protected Duration interval() { return interval; }
@@ -93,9 +104,12 @@ public abstract class Maintainer implements Runnable {
     public final void lockAndMaintain(boolean force) {
         if (!force && !jobControl.isActive(name())) return;
         log.log(Level.FINE, () -> "Running " + this.getClass().getSimpleName());
-        jobMetrics.recordRunOf(name());
+        jobMetrics.starting(name());
+        double successFactor = 0;
         try (var lock = jobControl.lockJob(name())) {
-            if (maintain()) jobMetrics.recordCompletionOf(name());
+            successFactor = maintain();
+            if (successFactor > 0.0)
+                jobMetrics.recordCompletionOf(name());
         } catch (UncheckedTimeoutException e) {
             if (ignoreCollision) {
                 jobMetrics.recordCompletionOf(name());
@@ -105,7 +119,7 @@ public abstract class Maintainer implements Runnable {
         } catch (Throwable e) {
             log.log(Level.WARNING, this + " failed. Will retry in " + interval, e);
         } finally {
-            jobMetrics.forward(name());
+            jobMetrics.completed(name(), successFactor);
         }
         log.log(Level.FINE, () -> "Finished " + this.getClass().getSimpleName());
     }
diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java
index 139a2901cd3..01560c050ff 100644
--- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java
+++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java
@@ -3,6 +3,8 @@ package com.yahoo.concurrent.maintenance;
 
 import org.junit.Test;
 
+import java.util.concurrent.atomic.AtomicLong;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -19,9 +21,8 @@ public class JobControlTest {
 
         String job1 = "Job1";
         String job2 = "Job2";
-        JobMetrics metrics = new JobMetrics((job, instant) -> {});
-        TestMaintainer maintainer1 = new TestMaintainer(job1, jobControl, metrics);
-        TestMaintainer maintainer2 = new TestMaintainer(job2, jobControl, metrics);
+        TestMaintainer maintainer1 = new TestMaintainer(job1, jobControl, new NoopJobMetrics());
+        TestMaintainer maintainer2 = new TestMaintainer(job2, jobControl, new NoopJobMetrics());
         assertEquals(2, jobControl.jobs().size());
         assertTrue(jobControl.jobs().contains(job1));
         assertTrue(jobControl.jobs().contains(job2));
@@ -62,7 +63,7 @@ public class JobControlTest {
     public void testJobControlMayDeactivateJobs() {
         JobControlStateMock state = new JobControlStateMock();
         JobControl jobControl = new JobControl(state);
-        TestMaintainer mockMaintainer = new TestMaintainer(null, jobControl, new JobMetrics((job, instant) -> {}));
+        TestMaintainer mockMaintainer = new TestMaintainer(null, jobControl, new NoopJobMetrics());
 
         assertTrue(jobControl.jobs().contains("TestMaintainer"));
 
@@ -80,4 +81,11 @@ public class JobControlTest {
         assertEquals(2, mockMaintainer.totalRuns());
     }
 
+    private static class NoopJobMetrics extends JobMetrics {
+
+        @Override
+        protected void recordCompletion(String job, Long incompleteRuns, double successFactor) { }
+
+    }
+
 }
diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java
index e881d4b3ff6..d2db380f4a1 100644
--- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java
+++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java
@@ -42,35 +42,45 @@ public class MaintainerTest {
 
     @Test
     public void success_metric() {
-        AtomicLong consecutiveFailures = new AtomicLong();
-        JobMetrics jobMetrics = new JobMetrics((job, count) -> consecutiveFailures.set(count));
+        TestJobMetrics jobMetrics = new TestJobMetrics();
         TestMaintainer maintainer = new TestMaintainer(null, jobControl, jobMetrics);
 
         // Maintainer fails twice in a row
         maintainer.successOnNextRun(false).run();
-        assertEquals(1, consecutiveFailures.get());
+        assertEquals(1, jobMetrics.consecutiveFailures.get());
         maintainer.successOnNextRun(false).run();
-        assertEquals(2, consecutiveFailures.get());
+        assertEquals(2, jobMetrics.consecutiveFailures.get());
 
         // Maintainer runs successfully
         maintainer.successOnNextRun(true).run();
-        assertEquals(0, consecutiveFailures.get());
+        assertEquals(0, jobMetrics.consecutiveFailures.get());
 
         // Maintainer runs successfully again
         maintainer.run();
-        assertEquals(0, consecutiveFailures.get());
+        assertEquals(0, jobMetrics.consecutiveFailures.get());
 
         // Maintainer throws
         maintainer.throwOnNextRun(new RuntimeException()).run();
-        assertEquals(1, consecutiveFailures.get());
+        assertEquals(1, jobMetrics.consecutiveFailures.get());
 
         // Maintainer recovers
         maintainer.throwOnNextRun(null).run();
-        assertEquals(0, consecutiveFailures.get());
+        assertEquals(0, jobMetrics.consecutiveFailures.get());
 
         // Lock exception is treated as a failure
         maintainer.throwOnNextRun(new UncheckedTimeoutException()).run();
-        assertEquals(1, consecutiveFailures.get());
+        assertEquals(1, jobMetrics.consecutiveFailures.get());
+    }
+
+    private static class TestJobMetrics extends JobMetrics {
+
+        AtomicLong consecutiveFailures = new AtomicLong();
+
+        @Override
+        protected void recordCompletion(String job, Long incompleteRuns, double successFactor) {
+            consecutiveFailures.set(incompleteRuns);
+        }
+
     }
 
 }
diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java
index 44a00a37a83..7424b17cab2 100644
--- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java
+++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java
@@ -33,10 +33,10 @@ class TestMaintainer extends Maintainer {
     }
 
     @Override
-    protected boolean maintain() {
+    protected double maintain() {
         if (exceptionToThrow != null) throw exceptionToThrow;
         totalRuns++;
-        return success;
+        return success ? 1.0 : 0.0;
     }
 
 }