Revert "Revert "Emit a success factor from maintainers""

This reverts commit cd1b747b4f65fa3a6ed6aace23235db7591638c5.

6 files changed, 34 insertions, 19 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
index 5519ffc1bdc..003b4fbb345 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
@@ -48,8 +48,9 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected boolean maintain() {
-        boolean success = true;
+    protected double maintain() {
+        int attempts = 0;
+        int failures = 0;
 
         try (var fileDownloader = new FileDownloader(connectionPool, downloadDirectory)) {
             for (var applicationId : applicationRepository.listApplications()) {
@@ -62,11 +63,12 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
                 log.fine(() -> "Verifying application package file reference " + applicationPackage + " for session " + sessionId);
 
                 if (applicationPackage != null) {
+                    attempts++;
                     if (! fileReferenceExistsOnDisk(downloadDirectory, applicationPackage)) {
                         log.fine(() -> "Downloading missing application package for application " + applicationId + " - session " + sessionId);
 
                         if (fileDownloader.getFile(applicationPackage).isEmpty()) {
-                            success = false;
+                            failures++;
                             log.warning("Failed to download application package for application " + applicationId + " - session " + sessionId);
                             continue;
                         }
@@ -75,7 +77,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
                 }
             }
         }
-        return success;
+        return asSuccessFactor(attempts, failures);
     }
 
     @Override
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
index 4938f34131e..e0f0a4b4099 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
@@ -35,14 +35,24 @@ public abstract class ConfigServerMaintainer extends Maintainer {
     ConfigServerMaintainer(ApplicationRepository applicationRepository, Curator curator, FlagSource flagSource,
                            Instant now, Duration interval) {
         super(null, interval, now, new JobControl(new JobControlFlags(curator, flagSource)),
-              jobMetrics(applicationRepository.metric()), cluster(curator), false);
+              new ConfigServerJobMetrics(applicationRepository.metric()), cluster(curator), false);
         this.applicationRepository = applicationRepository;
     }
 
-    private static JobMetrics jobMetrics(Metric metric) {
-        return new JobMetrics((job, consecutiveFailures) -> {
+    private static class ConfigServerJobMetrics extends JobMetrics {
+
+        private final Metric metric;
+
+        public ConfigServerJobMetrics(Metric metric) {
+            this.metric = metric;
+        }
+
+        @Override
+        protected void recordCompletion(String job, Long consecutiveFailures, double successFactor) {
             metric.set("maintenance.consecutiveFailures", consecutiveFailures, metric.createContext(Map.of("job", job)));
-        });
+            metric.set("maintenance.successFactor", successFactor, metric.createContext(Map.of("job", job)));
+        }
+
     }
 
     private static class JobControlFlags implements JobControlState {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
index b0876fb57e8..ca8db30c21f 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
@@ -33,9 +33,9 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected boolean maintain() {
+    protected double maintain() {
         applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge);
-        return true;
+        return 1.0;
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
index 971c2c20ae9..af9ea917aaf 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
@@ -22,6 +22,7 @@ import java.util.Comparator;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 import java.util.logging.Level;
@@ -51,8 +52,9 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected boolean maintain() {
-        AtomicBoolean success = new AtomicBoolean(true);
+    protected double maintain() {
+        AtomicInteger attempts = new AtomicInteger(0);
+        AtomicInteger failures = new AtomicInteger(0);
         for (Tenant tenant : applicationRepository.tenantRepository().getAllTenants()) {
             ApplicationCuratorDatabase database = tenant.getApplicationRepo().database();
             for (ApplicationId id : database.activeApplications())
@@ -60,6 +62,7 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
                                      .map(application -> application.getForVersionOrLatest(Optional.empty(), clock.instant()))
                                      .ifPresent(application -> {
                                          try {
+                                             attempts.incrementAndGet();
                                              applicationRepository.modifyReindexing(id, reindexing -> {
                                                  reindexing = withNewReady(reindexing, lazyGeneration(application), clock.instant());
                                                  reindexing = withOnlyCurrentData(reindexing, application);
@@ -68,11 +71,11 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
                                          }
                                          catch (RuntimeException e) {
                                              log.log(Level.INFO, "Failed to update reindexing status for " + id + ": " + Exceptions.toMessageString(e));
-                                             success.set(false);
+                                             failures.incrementAndGet();
                                          }
                                      });
         }
-        return success.get();
+        return asSuccessFactor(attempts.get(), failures.get());
     }
 
     private Supplier<Long> lazyGeneration(Application application) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
index 7482980e221..1f85dd4579d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
@@ -25,7 +25,7 @@ public class SessionsMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected boolean maintain() {
+    protected double maintain() {
         if (iteration % 10 == 0)
             log.log(Level.INFO, () -> "Running " + SessionsMaintainer.class.getSimpleName() + ", iteration "  + iteration);
 
@@ -38,7 +38,7 @@ public class SessionsMaintainer extends ConfigServerMaintainer {
         }
 
         iteration++;
-        return true;
+        return 1.0;
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
index 7c01045ee72..0a7df2c9d21 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
@@ -31,12 +31,12 @@ public class TenantsMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected boolean maintain() {
-        if ( ! applicationRepository.configserverConfig().hostedVespa()) return true;
+    protected double maintain() {
+        if ( ! applicationRepository.configserverConfig().hostedVespa()) return 1.0;
 
         Set<TenantName> tenants = applicationRepository.deleteUnusedTenants(ttlForUnusedTenant, clock.instant());
         if (tenants.size() > 0) log.log(Level.INFO, "Deleted tenants " + tenants);
-        return true;
+        return 1.0;
     }
 
 }