Emit QoS metric for all maintainers

7 files changed, 38 insertions, 9 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index bc9fa96f943..937cf4dfe7f 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -206,6 +206,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.metric = metric;
     }
 
+    public Clock clock() {
+        return clock;
+    }
+
+    public Metric metric() {
+        return metric;
+    }
+
     // ---------------- Deploying ----------------------------------------------------------------
 
     public PrepareResult prepare(Tenant tenant, long sessionId, PrepareParams prepareParams, Instant now) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
index ccbad4e21c7..92044eab5fe 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
@@ -50,8 +50,9 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected void maintain() {
-        if (! distributeApplicationPackage.value()) return;
+    protected boolean maintain() {
+        boolean success = true;
+        if (! distributeApplicationPackage.value()) return success;
 
         try (var fileDownloader = new FileDownloader(createConnectionPool(configserverConfig), downloadDirectory)) {
             for (var applicationId : applicationRepository.listApplications()) {
@@ -68,6 +69,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
                         log.fine(() -> "Downloading missing application package for application " + applicationId + " - session " + sessionId);
 
                         if (fileDownloader.getFile(applicationPackage).isEmpty()) {
+                            success = false;
                             log.warning("Failed to download application package for application " + applicationId + " - session " + sessionId);
                             continue;
                         }
@@ -76,6 +78,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
                 }
             }
         }
+        return success;
     }
 
     private void createLocalSessionIfMissing(ApplicationId applicationId, long sessionId) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
index 5369bbef366..007ca8dcf53 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
@@ -3,7 +3,9 @@ package com.yahoo.vespa.config.server.maintenance;
 
 import com.yahoo.concurrent.maintenance.JobControl;
 import com.yahoo.concurrent.maintenance.JobControlState;
+import com.yahoo.concurrent.maintenance.JobMetrics;
 import com.yahoo.concurrent.maintenance.Maintainer;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.path.Path;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.config.server.ApplicationRepository;
@@ -12,7 +14,9 @@ import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.ListFlag;
 
+import java.time.Clock;
 import java.time.Duration;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -26,16 +30,25 @@ public abstract class ConfigServerMaintainer extends Maintainer {
 
     ConfigServerMaintainer(ApplicationRepository applicationRepository, Curator curator, FlagSource flagSource,
                            Duration initialDelay, Duration interval) {
-        super(null, interval, initialDelay, new JobControl(new JobControlFlags(curator, flagSource)));
+        super(null, interval, initialDelay, new JobControl(new JobControlFlags(curator, flagSource)),
+              jobMetrics(applicationRepository.clock(), applicationRepository.metric()));
         this.applicationRepository = applicationRepository;
     }
 
+    private static JobMetrics jobMetrics(Clock clock, Metric metric) {
+        return new JobMetrics(clock, (job, instant) -> {
+            Duration sinceSuccess = Duration.between(instant, clock.instant());
+            metric.set("maintenance.secondsSinceSuccess", sinceSuccess.getSeconds(), metric.createContext(Map.of("job", job)));
+        });
+    }
+
     private static class JobControlFlags implements JobControlState {
 
         private static final Path root = Path.fromString("/configserver/v1/");
-        private static final Path lockRoot = root.append("locks");
 
+        private static final Path lockRoot = root.append("locks");
         private final Curator curator;
+
         private final ListFlag<String> inactiveJobsFlag;
 
         public JobControlFlags(Curator curator, FlagSource flagSource) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
index a6585be391c..adcaa3bb0e4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory;
 import com.yahoo.vespa.curator.Curator;
@@ -31,7 +32,8 @@ public class ConfigServerMaintenance extends AbstractComponent {
                                    ApplicationRepository applicationRepository,
                                    Curator curator,
                                    FileDistributionFactory fileDistributionFactory,
-                                   FlagSource flagSource) {
+                                   FlagSource flagSource,
+                                   Metric metric) {
         DefaultTimes defaults = new DefaultTimes(configserverConfig);
         // TODO: Disabled until we have application metadata
         //tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
index ed57be799c7..835122c043c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
@@ -35,8 +35,8 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected void maintain() {
+    protected boolean maintain() {
         applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge);
-
+        return true;
     }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
index 4adf287448d..77da56588ba 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
@@ -26,7 +26,7 @@ public class SessionsMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected void maintain() {
+    protected boolean maintain() {
         applicationRepository.deleteExpiredLocalSessions();
 
         // Expired remote sessions are sessions that belong to an application that have external deployments that
@@ -41,5 +41,7 @@ public class SessionsMaintainer extends ConfigServerMaintainer {
         int deleted = applicationRepository.deleteExpiredLocks(lockExpiryTime);
         if (deleted > 0)
             log.log(LogLevel.INFO, "Deleted " + deleted + " locks older than " + lockExpiryTime);
+
+        return true;
     }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
index 9a81d9f7547..d29eea842f5 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
@@ -28,8 +28,9 @@ public class TenantsMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected void maintain() {
+    protected boolean maintain() {
         applicationRepository.deleteUnusedTenants(ttlForUnusedTenant, clock.instant());
+        return true;
     }
 
 }