Use vespajlib maintenance package in node-repository

27 files changed, 96 insertions, 431 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 7045a47acf3..d612e8b102f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.yahoo.collections.ListMap;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.component.Version;
+import com.yahoo.concurrent.maintenance.JobControl;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Flavor;
@@ -23,7 +24,6 @@ import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList;
 import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions;
-import com.yahoo.vespa.hosted.provision.maintenance.JobControl;
 import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer;
 import com.yahoo.vespa.hosted.provision.maintenance.PeriodicApplicationMaintainer;
 import com.yahoo.vespa.hosted.provision.node.Agent;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
index b58fd3f67f1..06300e45e9e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
@@ -5,7 +5,6 @@ import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.jdisc.Metric;
-import java.util.logging.Level;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
 import java.time.Duration;
@@ -15,12 +14,13 @@ import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
 
 /**
  * @author bratseth
  * @author mpolden
  */
-public abstract class ApplicationMaintainer extends Maintainer {
+public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer {
 
     private final Deployer deployer;
     private final Metric metric;
@@ -91,8 +91,8 @@ public abstract class ApplicationMaintainer extends Maintainer {
     }
 
     @Override
-    public void deconstruct() {
-        super.deconstruct();
+    public void close() {
+        super.close();
         this.deploymentExecutor.shutdownNow();
         try {
             // Give deployments in progress some time to complete
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
index 94487547fab..da8a0a14171 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
@@ -1,27 +1,22 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.collections.Pair;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.jdisc.Metric;
-import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.applications.Application;
 import com.yahoo.vespa.hosted.provision.applications.Applications;
 import com.yahoo.vespa.hosted.provision.applications.Cluster;
-import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources;
 import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler;
 import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb;
 import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
 
 import java.time.Duration;
-import java.time.Instant;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -32,7 +27,7 @@ import java.util.stream.Collectors;
  *
  * @author bratseth
  */
-public class AutoscalingMaintainer extends Maintainer {
+public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
 
     private final Autoscaler autoscaler;
     private final Deployer deployer;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java
index b4c0d469272..abd1b477d4f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java
@@ -21,7 +21,7 @@ import java.util.*;
  *
  * @author mgimle
  */
-public class CapacityReportMaintainer extends Maintainer {
+public class CapacityReportMaintainer extends NodeRepositoryMaintainer {
 
     private final Metric metric;
     private final NodeRepository nodeRepository;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
index 906e6921f99..5c899f74bdb 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
@@ -38,7 +38,7 @@ import java.util.stream.IntStream;
 /**
  * @author freva
  */
-public class DynamicProvisioningMaintainer extends Maintainer {
+public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
 
     private static final Logger log = Logger.getLogger(DynamicProvisioningMaintainer.class.getName());
     private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision");
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
index ca576be8a69..dc5155312e7 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
@@ -17,7 +17,7 @@ import java.util.logging.Logger;
  *
  * @author bratseth
  */
-public abstract class Expirer extends Maintainer {
+public abstract class Expirer extends NodeRepositoryMaintainer {
 
     protected static final Logger log = Logger.getLogger(Expirer.class.getName());
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
index af37aa5becf..7e02eedc572 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
@@ -45,7 +45,7 @@ import java.util.stream.Collectors;
  * @author bratseth
  * @author mpolden
  */
-public class FailedExpirer extends Maintainer {
+public class FailedExpirer extends NodeRepositoryMaintainer {
 
     private static final Logger log = Logger.getLogger(FailedExpirer.class.getName());
     private static final int maxAllowedFailures = 5; // Stop recycling nodes after this number of failures
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
index 932c435221c..b933e549357 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
@@ -14,7 +14,7 @@ import java.util.logging.Logger;
  *
  * @author freva
  */
-public class InfrastructureProvisioner extends Maintainer {
+public class InfrastructureProvisioner extends NodeRepositoryMaintainer {
 
     private static final Logger logger = Logger.getLogger(InfrastructureProvisioner.class.getName());
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java
deleted file mode 100644
index 2d641ef57ab..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java
+++ /dev/null
@@ -1,69 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.maintenance;
-
-import com.yahoo.vespa.curator.Lock;
-import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentSkipListMap;
-
-/**
- * Provides status and control over running maintenance jobs.
- *
- * This is multi-thread safe.
- * 
- * @author bratseth
- */
-public class JobControl {
-
-    /** This is not stored in ZooKeeper as all nodes start all jobs */
-    private final Map<String, Maintainer> startedJobs = new ConcurrentSkipListMap<>();
-
-    /** Used to store deactivation in ZooKeeper to make changes take effect on all nodes */
-    private final CuratorDatabaseClient db;
-
-    public JobControl(CuratorDatabaseClient db) {
-        this.db = db;
-    }
-    
-    /** Notifies this that a job was started */
-    public void started(String jobSimpleClassName, Maintainer maintainer) {
-        startedJobs.put(jobSimpleClassName, maintainer);
-    }
-
-    /**
-     * Returns a snapshot of the set of jobs started on this system (whether deactivated or not).
-     * Each job is represented by its simple (omitting package) class name.
-     */
-    public Set<String> jobs() { return Collections.unmodifiableSet(startedJobs.keySet()); }
-
-    /** Returns a snapshot containing the currently inactive jobs in this */
-    public Set<String> inactiveJobs() { return db.readInactiveJobs(); }
-    
-    /** Returns true if this job is not currently deactivated */
-    public boolean isActive(String jobSimpleClassName) {
-        return  ! inactiveJobs().contains(jobSimpleClassName);
-    }
-
-    /** Set a job active or inactive */
-    public void setActive(String jobSimpleClassName, boolean active) {
-        try (Lock lock = db.lockInactiveJobs()) {
-            Set<String> inactiveJobs = db.readInactiveJobs();
-            if (active)
-                inactiveJobs.remove(jobSimpleClassName);
-            else
-                inactiveJobs.add(jobSimpleClassName);
-            db.writeInactiveJobs(inactiveJobs);
-        }
-    }
-
-    /** Run given job (inactive or not) immediately */
-    public void run(String jobSimpleClassName) {
-        var job = startedJobs.get(jobSimpleClassName);
-        if (job == null) throw new IllegalArgumentException("No such job '" + jobSimpleClassName + "'");
-        job.runWithLock();
-    }
-    
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
index 55152f52744..d793874119a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
@@ -30,7 +30,7 @@ import java.util.stream.Collectors;
  *
  * @author mpolden
  */
-public class LoadBalancerExpirer extends Maintainer {
+public class LoadBalancerExpirer extends NodeRepositoryMaintainer {
 
     private static final Duration reservedExpiry = Duration.ofHours(1);
     private static final Duration inactiveExpiry = Duration.ofHours(1);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
deleted file mode 100644
index 17c73282b4b..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
+++ /dev/null
@@ -1,119 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.maintenance;
-
-import com.google.common.util.concurrent.UncheckedTimeoutException;
-import com.yahoo.component.AbstractComponent;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.vespa.hosted.provision.Node;
-import com.yahoo.vespa.hosted.provision.NodeRepository;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-/**
- * A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo.
- *
- * @author bratseth
- */
-public abstract class Maintainer extends AbstractComponent implements Runnable {
-
-    protected final Logger log = Logger.getLogger(this.getClass().getName());
-
-    private final NodeRepository nodeRepository;
-    private final Duration interval;
-    private final JobControl jobControl;
-
-    private final ScheduledExecutorService service;
-
-    public Maintainer(NodeRepository nodeRepository, Duration interval) {
-        this.nodeRepository = nodeRepository;
-        this.interval = interval;
-        this.jobControl = nodeRepository.jobControl();
-
-        HostName hostname = HostName.from(com.yahoo.net.HostName.getLocalhost());
-        long delay = staggeredDelay(nodeRepository.database().cluster(), hostname, nodeRepository.clock().instant(), interval);
-        service = new ScheduledThreadPoolExecutor(1, r -> new Thread(r, name() + "-worker"));
-        service.scheduleAtFixedRate(this, delay, interval.toMillis(), TimeUnit.MILLISECONDS);
-        jobControl.started(name(), this);
-    }
-
-    /** Returns the node repository */
-    protected NodeRepository nodeRepository() { return nodeRepository; }
-
-    protected static Duration min(Duration a, Duration b) {
-        return a.toMillis() < b.toMillis() ? a : b;
-    }
-
-    /** Returns the interval at which this job is set to run */
-    protected Duration interval() { return interval; }
-
-    @Override
-    public void run() {
-        try {
-            if (jobControl.isActive(name())) {
-                runWithLock();
-            }
-        } catch (UncheckedTimeoutException ignored) {
-            // Another config server or operator is running this job
-        } catch (Throwable e) {
-            log.log(Level.WARNING, this + " failed. Will retry in " + interval.toMinutes() + " minutes", e);
-        }
-    }
-
-    @Override
-    public void deconstruct() {
-        var timeout = Duration.ofSeconds(30);
-        service.shutdown();
-        try {
-            if (!service.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS)) {
-                log.log(Level.WARNING, "Maintainer " + name() + " failed to shutdown " +
-                                       "within " + timeout);
-            }
-        } catch (InterruptedException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    /** Returns the simple name of this job */
-    @Override
-    public final String toString() { return name(); }
-
-    /** Run this while holding the job lock */
-    public void runWithLock() {
-        try (var lock = nodeRepository.database().lockMaintenanceJob(name())) {
-            maintain();
-        }
-    }
-
-    /** Called once each time this maintenance job should run */
-    protected abstract void maintain();
-    
-    private String name() { return this.getClass().getSimpleName(); }
-
-    /** A utility to group active tenant nodes by application */
-    protected Map<ApplicationId, List<Node>> activeNodesByApplication() {
-        return nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList()
-                               .stream()
-                               .filter(node -> ! node.allocation().get().owner().instance().isTester())
-                               .collect(Collectors.groupingBy(node -> node.allocation().get().owner()));
-    }
-
-    static long staggeredDelay(List<HostName> cluster, HostName host, Instant now, Duration interval) {
-        if ( ! cluster.contains(host))
-            return interval.toMillis();
-
-        long offset = cluster.indexOf(host) * interval.toMillis() / cluster.size();
-        return Math.floorMod(offset - now.toEpochMilli(), interval.toMillis());
-    }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index 098d706bf05..c631de5f17b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -34,7 +34,7 @@ import static com.yahoo.vespa.hosted.provision.Node.State.active;
 /**
  * @author oyving
  */
-public class MetricsReporter extends Maintainer {
+public class MetricsReporter extends NodeRepositoryMaintainer {
 
     private final Metric metric;
     private final Orchestrator orchestrator;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index cffbfec8f4b..7ce674cbbbf 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -31,7 +31,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -45,7 +44,7 @@ import static java.util.stream.Collectors.counting;
  * @author bratseth
  * @author mpolden
  */
-public class NodeFailer extends Maintainer {
+public class NodeFailer extends NodeRepositoryMaintainer {
 
     private static final Logger log = Logger.getLogger(NodeFailer.class.getName());
     private static final Duration nodeRequestInterval = Duration.ofMinutes(10);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
index adc1911a169..eb2b46dd53e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
@@ -16,7 +16,7 @@ import java.util.logging.Level;
  *
  * @author bratseth
  */
-public class NodeMetricsDbMaintainer extends Maintainer {
+public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer {
 
     private static final int maxWarningsPerInvocation = 2;
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
index d1e15c78a47..c78ed72ff42 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
@@ -26,7 +26,7 @@ import java.util.stream.Collectors;
  * 
  * @author bratseth
  */
-public class NodeRebooter extends Maintainer {
+public class NodeRebooter extends NodeRepositoryMaintainer {
 
     private final IntFlag rebootIntervalInDays;
     private final Clock clock;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java
new file mode 100644
index 00000000000..8368569cda0
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java
@@ -0,0 +1,44 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.concurrent.maintenance.Maintainer;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo.
+ *
+ * @author bratseth
+ */
+public abstract class NodeRepositoryMaintainer extends Maintainer {
+
+    private final NodeRepository nodeRepository;
+
+    public NodeRepositoryMaintainer(NodeRepository nodeRepository, Duration interval) {
+        super(null, interval, nodeRepository.clock().instant(), nodeRepository.jobControl(), nodeRepository.database().cluster());
+        this.nodeRepository = nodeRepository;
+    }
+
+    protected static Duration min(Duration a, Duration b) {
+        return a.toMillis() < b.toMillis() ? a : b;
+    }
+
+    /** Returns the node repository */
+    protected NodeRepository nodeRepository() { return nodeRepository; }
+
+    /** A utility to group active tenant nodes by application */
+    protected Map<ApplicationId, List<Node>> activeNodesByApplication() {
+        return nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList()
+                               .stream()
+                               .filter(node -> ! node.allocation().get().owner().instance().isTester())
+                               .collect(Collectors.groupingBy(node -> node.allocation().get().owner()));
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 9e954e0a1dd..a50e6d10696 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -101,26 +101,26 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
 
     @Override
     public void deconstruct() {
-        nodeFailer.deconstruct();
-        periodicApplicationMaintainer.deconstruct();
-        operatorChangeApplicationMaintainer.deconstruct();
-        reservationExpirer.deconstruct();
-        inactiveExpirer.deconstruct();
-        retiredExpirer.deconstruct();
-        failedExpirer.deconstruct();
-        dirtyExpirer.deconstruct();
-        nodeRebooter.deconstruct();
-        capacityReportMaintainer.deconstruct();
-        provisionedExpirer.deconstruct();
-        metricsReporter.deconstruct();
-        infrastructureProvisioner.deconstruct();
-        loadBalancerExpirer.ifPresent(Maintainer::deconstruct);
-        dynamicProvisioningMaintainer.ifPresent(Maintainer::deconstruct);
-        osUpgradeActivator.deconstruct();
-        rebalancer.deconstruct();
-        nodeMetricsDbMaintainer.deconstruct();
-        autoscalingMaintainer.deconstruct();
-        scalingSuggestionsMaintainer.deconstruct();
+        nodeFailer.close();
+        periodicApplicationMaintainer.close();
+        operatorChangeApplicationMaintainer.close();
+        reservationExpirer.close();
+        inactiveExpirer.close();
+        retiredExpirer.close();
+        failedExpirer.close();
+        dirtyExpirer.close();
+        nodeRebooter.close();
+        capacityReportMaintainer.close();
+        provisionedExpirer.close();
+        metricsReporter.close();
+        infrastructureProvisioner.close();
+        loadBalancerExpirer.ifPresent(NodeRepositoryMaintainer::close);
+        dynamicProvisioningMaintainer.ifPresent(NodeRepositoryMaintainer::close);
+        osUpgradeActivator.close();
+        rebalancer.close();
+        nodeMetricsDbMaintainer.close();
+        autoscalingMaintainer.close();
+        scalingSuggestionsMaintainer.close();
     }
 
     private static Optional<NodeFailer.ThrottlePolicy> throttlePolicyFromEnv() {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java
index c22bda8a5cf..b98b03671c4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java
@@ -15,7 +15,7 @@ import java.time.Duration;
  *
  * @author mpolden
  */
-public class OsUpgradeActivator extends Maintainer {
+public class OsUpgradeActivator extends NodeRepositoryMaintainer {
 
     public OsUpgradeActivator(NodeRepository nodeRepository, Duration interval) {
         super(nodeRepository, interval);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
index 65a24ea0ec8..0ba3765f470 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
@@ -22,7 +22,7 @@ import java.util.Optional;
 /**
  * @author bratseth
  */
-public class Rebalancer extends Maintainer {
+public class Rebalancer extends NodeRepositoryMaintainer {
 
     private final Deployer deployer;
     private final HostResourcesCalculator hostResourcesCalculator;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
index 3c01c8bc23c..73869fb098d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
@@ -24,7 +24,7 @@ import java.util.stream.Collectors;
  *
  * @author hakon
  */
-public class RetiredExpirer extends Maintainer {
+public class RetiredExpirer extends NodeRepositoryMaintainer {
 
     private final Deployer deployer;
     private final Metric metric;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java
index 6296543aea4..332126690be 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java
@@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.applications.Application;
 import com.yahoo.vespa.hosted.provision.applications.Applications;
 import com.yahoo.vespa.hosted.provision.applications.Cluster;
-import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources;
 import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler;
 import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb;
 import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
@@ -26,7 +25,7 @@ import java.util.stream.Collectors;
  *
  * @author bratseth
  */
-public class ScalingSuggestionsMaintainer extends Maintainer {
+public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer {
 
     private final Autoscaler autoscaler;
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index 2b64037a8b9..4475014172e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.provision.persistence;
 
 import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.component.Version;
+import com.yahoo.concurrent.maintenance.JobControl;
+import com.yahoo.concurrent.maintenance.StringSetSerializer;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationLockException;
 import com.yahoo.config.provision.DockerImage;
@@ -57,7 +59,7 @@ import static java.util.stream.Collectors.toMap;
  *
  * @author bratseth
  */
-public class CuratorDatabaseClient {
+public class CuratorDatabaseClient implements JobControl.Db {
 
     private static final Logger log = Logger.getLogger(CuratorDatabaseClient.class.getName());
 
@@ -90,8 +92,8 @@ public class CuratorDatabaseClient {
         initZK();
     }
 
-    public List<HostName> cluster() {
-        return db.cluster();
+    public List<String> cluster() {
+        return db.cluster().stream().map(HostName::value).collect(Collectors.toUnmodifiableList());
     }
 
     private void initZK() {
@@ -452,10 +454,12 @@ public class CuratorDatabaseClient {
 
     // Maintenance jobs -----------------------------------------------------------
 
+    @Override
     public Lock lockMaintenanceJob(String jobName) {
         return db.lock(lockPath.append("maintenanceJobLocks").append(jobName), defaultLockTimeout);
     }
 
+    @Override
     public Set<String> readInactiveJobs() {
         try {
             return read(inactiveJobsPath, stringSetSerializer::fromJson).orElseGet(HashSet::new);
@@ -467,6 +471,7 @@ public class CuratorDatabaseClient {
         }
     }
 
+    @Override
     public void writeInactiveJobs(Set<String> inactiveJobs) {
         NestedTransaction transaction = new NestedTransaction();
         CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction);
@@ -474,7 +479,8 @@ public class CuratorDatabaseClient {
                                                          stringSetSerializer.toJson(inactiveJobs)));
         transaction.commit();
     }
-    
+
+    @Override
     public Lock lockInactiveJobs() {
         return db.lock(lockPath.append("inactiveJobsLock"), defaultLockTimeout);
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java
deleted file mode 100644
index 1149b15a2b0..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.persistence;
-
-import com.yahoo.slime.ArrayTraverser;
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.Inspector;
-import com.yahoo.slime.Slime;
-import com.yahoo.slime.SlimeUtils;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * Serialization of a set of strings to/from Json bytes using Slime.
- * 
- * The set is serialized as an array of string.
- * 
- * @author bratseth
- */
-public class StringSetSerializer {
-
-    public byte[] toJson(Set<String> stringSet) {
-        try {
-            Slime slime = new Slime();
-            Cursor array = slime.setArray();
-            for (String element : stringSet)
-                array.addString(element);
-            return SlimeUtils.toJsonBytes(slime);
-        }
-        catch (IOException e) {
-            throw new RuntimeException("Serialization of a string set failed", e);
-        }
-        
-    }
-
-    public Set<String> fromJson(byte[] data) {
-        Inspector inspector = SlimeUtils.jsonToSlime(data).get();
-        Set<String> stringSet = new HashSet<>();
-        inspector.traverse((ArrayTraverser) (index, name) -> stringSet.add(name.asString()));
-        return stringSet;
-    }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java
index 3b6de41f142..eb425095978 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java
@@ -1,11 +1,11 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.restapi;
 
+import com.yahoo.concurrent.maintenance.JobControl;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.JsonFormat;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.hosted.provision.maintenance.JobControl;
 
 import java.io.IOException;
 import java.io.OutputStream;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java
deleted file mode 100644
index 396fcd67034..00000000000
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.maintenance;
-
-import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.NodeRepositoryTester;
-import org.junit.Test;
-
-import java.time.Duration;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author bratseth
- */
-public class JobControlTest {
-    
-    @Test
-    public void testJobControl() {
-        NodeRepositoryTester tester = new NodeRepositoryTester();
-        JobControl jobControl = new JobControl(tester.nodeRepository().database());
-
-        MockMaintainer maintainer1 = new MockMaintainer(tester.nodeRepository());
-        MockMaintainer maintainer2 = new MockMaintainer(tester.nodeRepository());
-        assertTrue(jobControl.jobs().isEmpty());
-
-        String job1 = "Job1";
-        String job2 = "Job2";
-        
-        jobControl.started(job1, maintainer1);
-        jobControl.started(job2, maintainer2);
-        assertEquals(2, jobControl.jobs().size());
-        assertTrue(jobControl.jobs().contains(job1));
-        assertTrue(jobControl.jobs().contains(job2));
-        
-        assertTrue(jobControl.isActive(job1));
-        assertTrue(jobControl.isActive(job2));
-        
-        jobControl.setActive(job1, false);
-        assertFalse(jobControl.isActive(job1));
-        assertTrue(jobControl.isActive(job2));
-
-        jobControl.setActive(job2, false);
-        assertFalse(jobControl.isActive(job1));
-        assertFalse(jobControl.isActive(job2));
-
-        jobControl.setActive(job1, true);
-        assertTrue(jobControl.isActive(job1));
-        assertFalse(jobControl.isActive(job2));
-
-        jobControl.setActive(job2, true);
-        assertTrue(jobControl.isActive(job1));
-        assertTrue(jobControl.isActive(job2));
-
-        // Run jobs on-demand
-        jobControl.run(job1);
-        jobControl.run(job1);
-        assertEquals(2, maintainer1.maintenanceInvocations);
-        jobControl.run(job2);
-        assertEquals(1, maintainer2.maintenanceInvocations);
-
-        // Running jobs on-demand ignores inactive flag
-        jobControl.setActive(job1, false);
-        jobControl.run(job1);
-        assertEquals(3, maintainer1.maintenanceInvocations);
-    }
-    
-    @Test
-    public void testJobControlMayDeactivateJobs() {
-        NodeRepositoryTester tester = new NodeRepositoryTester();
-        JobControl jobControl = tester.nodeRepository().jobControl();
-        MockMaintainer mockMaintainer = new MockMaintainer(tester.nodeRepository());
-        
-        assertTrue(jobControl.jobs().contains("MockMaintainer"));
-
-        assertEquals(0, mockMaintainer.maintenanceInvocations);
-
-        mockMaintainer.run();
-        assertEquals(1, mockMaintainer.maintenanceInvocations);
-
-        jobControl.setActive("MockMaintainer", false);
-        mockMaintainer.run();
-        assertEquals(1, mockMaintainer.maintenanceInvocations);
-
-        jobControl.setActive("MockMaintainer", true);
-        mockMaintainer.run();
-        assertEquals(2, mockMaintainer.maintenanceInvocations);
-    }
-    
-    private static class MockMaintainer extends Maintainer {
-        
-        int maintenanceInvocations = 0;
-        
-        private MockMaintainer(NodeRepository nodeRepository) {
-            super(nodeRepository, Duration.ofHours(1));
-        }
-
-        @Override
-        protected void maintain() {
-            maintenanceInvocations++;
-        }
-
-    }
-    
-}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java
deleted file mode 100644
index 7ce64093491..00000000000
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.maintenance;
-
-import com.yahoo.config.provision.HostName;
-import org.junit.Test;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author freva
- */
-public class MaintainerTest {
-
-    @Test
-    public void staggering() {
-        List<HostName> cluster = Arrays.asList(HostName.from("cfg1"), HostName.from("cfg2"), HostName.from("cfg3"));
-        Duration interval = Duration.ofMillis(300);
-        Instant now = Instant.ofEpochMilli(1000);
-        assertEquals(200, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
-        assertEquals(  0, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
-        assertEquals(100, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
-
-        now = Instant.ofEpochMilli(1001);
-        assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
-        assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
-        assertEquals( 99, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
-
-        now = Instant.ofEpochMilli(1101);
-        assertEquals( 99, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
-        assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
-        assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
-
-        assertEquals(300, Maintainer.staggeredDelay(cluster, HostName.from("cfg0"), now, interval));
-    }
-}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index 63a22cc3029..91e4899e079 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -71,7 +71,7 @@ public class PeriodicApplicationMaintainerTest {
 
     @After
     public void after() {
-        this.fixture.maintainer.deconstruct();
+        this.fixture.maintainer.close();
     }
 
     @Test(timeout = 60_000)