Persist job control

18 files changed, 194 insertions, 54 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 51172c7045d..df5675d6287 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -67,7 +67,7 @@ import java.util.stream.Collectors;
 // Nodes might have an application assigned in dirty.
 public class NodeRepository extends AbstractComponent {
 
-    private final CuratorDatabaseClient zkClient;
+    private final CuratorDatabaseClient db;
     private final Curator curator;
     private final Clock clock;
     private final NodeFlavors flavors;
@@ -87,7 +87,7 @@ public class NodeRepository extends AbstractComponent {
      * which will be used for time-sensitive decisions.
      */
     public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver) {
-        this.zkClient = new CuratorDatabaseClient(flavors, curator, clock, zone);
+        this.db = new CuratorDatabaseClient(flavors, curator, clock, zone);
         this.curator = curator;
         this.clock = clock;
         this.flavors = flavors;
@@ -95,9 +95,12 @@ public class NodeRepository extends AbstractComponent {
 
         // read and write all nodes to make sure they are stored in the latest version of the serialized format
         for (Node.State state : Node.State.values())
-            zkClient.writeTo(state, zkClient.getNodes(state), Agent.system, Optional.empty());
+            db.writeTo(state, db.getNodes(state), Agent.system, Optional.empty());
     }
 
+    /** Returns the curator database client used by this */
+    public CuratorDatabaseClient database() { return db; }
+    
     // ---------------- Query API ----------------------------------------------------------------
 
     /**
@@ -108,7 +111,7 @@ public class NodeRepository extends AbstractComponent {
      * @return the node, or empty if it was not found in any of the given states
      */
     public Optional<Node> getNode(String hostname, Node.State ... inState) {
-        return zkClient.getNode(hostname, inState);
+        return db.getNode(hostname, inState);
     }
 
     /**
@@ -118,7 +121,7 @@ public class NodeRepository extends AbstractComponent {
      * @return the node, or empty if it was not found in any of the given states
      */
     public List<Node> getNodes(Node.State ... inState) {
-        return zkClient.getNodes(inState).stream().collect(Collectors.toList());
+        return db.getNodes(inState).stream().collect(Collectors.toList());
     }
     /**
      * Finds and returns the nodes of the given type in any of the given states.
@@ -128,7 +131,7 @@ public class NodeRepository extends AbstractComponent {
      * @return the node, or empty if it was not found in any of the given states
      */
     public List<Node> getNodes(NodeType type, Node.State ... inState) {
-        return zkClient.getNodes(inState).stream().filter(node -> node.type().equals(type)).collect(Collectors.toList());
+        return db.getNodes(inState).stream().filter(node -> node.type().equals(type)).collect(Collectors.toList());
     }
 
     /**
@@ -138,16 +141,16 @@ public class NodeRepository extends AbstractComponent {
      * @return List of child nodes
      */
     public List<Node> getChildNodes(String hostname) {
-        return zkClient.getNodes().stream()
+        return db.getNodes().stream()
                 .filter(node -> node.parentHostname()
                         .map(parentHostname -> parentHostname.equals(hostname))
                         .orElse(false))
                 .collect(Collectors.toList());
     }
 
-    public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return zkClient.getNodes(id, inState); }
-    public List<Node> getInactive() { return zkClient.getNodes(Node.State.inactive); }
-    public List<Node> getFailed() { return zkClient.getNodes(Node.State.failed); }
+    public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return db.getNodes(id, inState); }
+    public List<Node> getInactive() { return db.getNodes(Node.State.inactive); }
+    public List<Node> getFailed() { return db.getNodes(Node.State.failed); }
 
     /**
      * Returns a set of nodes that should be trusted by the given node.
@@ -227,7 +230,7 @@ public class NodeRepository extends AbstractComponent {
 
     /** Get default flavor override for an application, if present. */
     public Optional<String> getDefaultFlavorOverride(ApplicationId applicationId) {
-        return zkClient.getDefaultFlavorForApplication(applicationId);
+        return db.getDefaultFlavorForApplication(applicationId);
     }
 
     // ----------------- Node lifecycle -----------------------------------------------------------
@@ -254,7 +257,7 @@ public class NodeRepository extends AbstractComponent {
                 throw new IllegalArgumentException("Cannot add " + node.hostname() + ": A node with this name already exists");
         }
         try (Mutex lock = lockUnallocated()) {
-            return zkClient.addNodes(nodes);
+            return db.addNodes(nodes);
         }
     }
 
@@ -264,7 +267,7 @@ public class NodeRepository extends AbstractComponent {
             if (node.state() != Node.State.dirty)
                 throw new IllegalArgumentException("Can not set " + node + " ready. It is not dirty.");
         try (Mutex lock = lockUnallocated()) {
-            return zkClient.writeTo(Node.State.ready, nodes, Agent.system, Optional.empty());
+            return db.writeTo(Node.State.ready, nodes, Agent.system, Optional.empty());
         }
     }
 
@@ -278,12 +281,12 @@ public class NodeRepository extends AbstractComponent {
 
     /** Reserve nodes. This method does <b>not</b> lock the node repository */
     public List<Node> reserve(List<Node> nodes) { 
-        return zkClient.writeTo(Node.State.reserved, nodes, Agent.application, Optional.empty()); 
+        return db.writeTo(Node.State.reserved, nodes, Agent.application, Optional.empty()); 
     }
 
     /** Activate nodes. This method does <b>not</b> lock the node repository */
     public List<Node> activate(List<Node> nodes, NestedTransaction transaction) {
-        return zkClient.writeTo(Node.State.active, nodes, Agent.application, Optional.empty(), transaction);
+        return db.writeTo(Node.State.active, nodes, Agent.application, Optional.empty(), transaction);
     }
 
     /**
@@ -303,9 +306,9 @@ public class NodeRepository extends AbstractComponent {
 
     public void deactivate(ApplicationId application, NestedTransaction transaction) {
         try (Mutex lock = lock(application)) {
-            zkClient.writeTo(Node.State.inactive,
-                             zkClient.getNodes(application, Node.State.reserved, Node.State.active),
-                             Agent.application, Optional.empty(), transaction
+            db.writeTo(Node.State.inactive,
+                       db.getNodes(application, Node.State.reserved, Node.State.active),
+                       Agent.application, Optional.empty(), transaction
             );
         }
     }
@@ -316,7 +319,7 @@ public class NodeRepository extends AbstractComponent {
      * This method does <b>not</b> lock
      */
     public List<Node> deactivate(List<Node> nodes, NestedTransaction transaction) {
-        return zkClient.writeTo(Node.State.inactive, nodes, Agent.application, Optional.empty(), transaction);
+        return db.writeTo(Node.State.inactive, nodes, Agent.application, Optional.empty(), transaction);
     }
 
     /** Move nodes to the dirty state */
@@ -326,7 +329,7 @@ public class NodeRepository extends AbstractComponent {
 
     /** Move a single node to the dirty state */
     public Node setDirty(Node node) {
-        return zkClient.writeTo(Node.State.dirty, node, Agent.system, Optional.empty());
+        return db.writeTo(Node.State.dirty, node, Agent.system, Optional.empty());
     }
 
     /**
@@ -420,7 +423,7 @@ public class NodeRepository extends AbstractComponent {
                                                            "It has the same cluster and index as an existing node");
                 }
             }
-            return zkClient.writeTo(toState, node, agent, reason);
+            return db.writeTo(toState, node, agent, reason);
         }
     }
 
@@ -434,7 +437,7 @@ public class NodeRepository extends AbstractComponent {
         if ( ! nodeToRemove.isPresent()) return false;
 
         try (Mutex lock = lock(nodeToRemove.get())) {
-            return zkClient.removeNode(nodeToRemove.get().state(), hostname);
+            return db.removeNode(nodeToRemove.get().state(), hostname);
         }
     }
 
@@ -460,8 +463,8 @@ public class NodeRepository extends AbstractComponent {
      *
      * @return the written node for convenience
      */
-    public Node write(Node node) { return zkClient.writeTo(node.state(), node,
-                                                           Agent.system, Optional.empty()); }
+    public Node write(Node node) { return db.writeTo(node.state(), node,
+                                                     Agent.system, Optional.empty()); }
 
     /**
      * Writes these nodes after they have changed some internal state but NOT changed their state field.
@@ -478,7 +481,7 @@ public class NodeRepository extends AbstractComponent {
             if ( node.state() != state)
                 throw new IllegalArgumentException("Multiple states: " + node.state() + " and " + state);
         }
-        return zkClient.writeTo(state, nodes, Agent.system, Optional.empty());
+        return db.writeTo(state, nodes, Agent.system, Optional.empty());
     }
 
     /**
@@ -493,7 +496,7 @@ public class NodeRepository extends AbstractComponent {
         ListMap<ApplicationId, Node> allocatedNodes = new ListMap<>();
 
         // Group matching nodes by the lock needed
-        for (Node node : zkClient.getNodes()) {
+        for (Node node : db.getNodes()) {
             if ( ! filter.matches(node)) continue;
             if (node.allocation().isPresent())
                 allocatedNodes.put(node.allocation().get().owner(), node);
@@ -531,13 +534,13 @@ public class NodeRepository extends AbstractComponent {
     public Clock clock() { return clock; }
 
     /** Create a lock which provides exclusive rights to making changes to the given application */
-    public Mutex lock(ApplicationId application) { return zkClient.lock(application); }
+    public Mutex lock(ApplicationId application) { return db.lock(application); }
 
     /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */
-    public Mutex lock(ApplicationId application, Duration timeout) { return zkClient.lock(application, timeout); }
+    public Mutex lock(ApplicationId application, Duration timeout) { return db.lock(application, timeout); }
 
     /** Create a lock which provides exclusive rights to changing the set of ready nodes */
-    public Mutex lockUnallocated() { return zkClient.lockInactive(); }
+    public Mutex lockUnallocated() { return db.lockInactive(); }
 
     /** Acquires the appropriate lock for this node */
     private Mutex lock(Node node) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java
index 471e76fb655..93c626c8b81 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java
@@ -1,6 +1,8 @@
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import java.util.Collections;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorMutex;
+
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.ConcurrentSkipListSet;
@@ -12,9 +14,16 @@ import java.util.concurrent.ConcurrentSkipListSet;
  * @author bratseth
  */
 public class JobControl {
-    
+
+    /** This is not stored in ZooKeeper as all nodes start all jobs */
     private Set<String> startedJobs = new ConcurrentSkipListSet<>();
-    private Set<String> deactivatedJobs = new ConcurrentSkipListSet<>();
+
+    /** Used to store deactivation in ZooKeeper to make changes take effect on all nodes */
+    private final CuratorDatabaseClient db;
+    
+    public JobControl(CuratorDatabaseClient db) {
+        this.db = db;
+    }
     
     /** Notifies this that a job was started */
     public void started(String jobSimpleClassName) {
@@ -29,15 +38,19 @@ public class JobControl {
     
     /** Returns true if this job is not currently deactivated */
     public boolean isActive(String jobSimpleClassName) {
-        return ! deactivatedJobs.contains(jobSimpleClassName);
+        return  ! db.readDeactivatedJobs().contains(jobSimpleClassName);
     }
 
     /** Set a job active or inactive */
     public void setActive(String jobSimpleClassName, boolean active) {
-        if (active)
-            deactivatedJobs.remove(jobSimpleClassName);
-        else
-            deactivatedJobs.add(jobSimpleClassName);
+        try (CuratorMutex lock = db.lockDeactivatedJobs()) {
+            Set<String> deactivatedJobs = db.readDeactivatedJobs();
+            if (active)
+                deactivatedJobs.remove(jobSimpleClassName);
+            else
+                deactivatedJobs.add(jobSimpleClassName);
+            db.writeDeactivatedJobs(deactivatedJobs);
+        }
     }
     
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index ec74713af22..970dbde9cc8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -41,7 +41,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     private final NodeRebooter nodeRebooter;
     private final MetricsReporter metricsReporter;
     
-    private final JobControl jobControl = new JobControl();
+    private final JobControl jobControl;
 
     @Inject
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Curator curator,
@@ -54,6 +54,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, 
                                      Zone zone, Clock clock, Orchestrator orchestrator, Metric metric) {
         DefaultTimes defaults = new DefaultTimes(zone.environment());
+        jobControl = new JobControl(nodeRepository.database());
         nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, durationFromEnv("fail_grace").orElse(defaults.failGrace), clock, orchestrator, throttlePolicyFromEnv("throttle_policy").orElse(defaults.throttlePolicy), jobControl);
         periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, nodeRepository, durationFromEnv("periodic_redeploy_interval").orElse(defaults.periodicRedeployInterval), jobControl);
         operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, nodeRepository, clock, durationFromEnv("operator_change_redeploy_interval").orElse(defaults.operatorChangeRedeployInterval), jobControl);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index 5961b6cc6f8..08d56c9cb42 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import java.util.logging.Logger;
 
 /**
@@ -45,6 +46,8 @@ public class CuratorDatabaseClient {
     private static final Duration defaultLockTimeout = Duration.ofMinutes(1);
 
     private final NodeSerializer nodeSerializer;
+    
+    private final StringSetSerializer stringSetSerializer = new StringSetSerializer();
 
     private final CuratorDatabase curatorDatabase;
 
@@ -65,6 +68,7 @@ public class CuratorDatabaseClient {
         curatorDatabase.create(root);
         for (Node.State state : Node.State.values())
             curatorDatabase.create(toPath(state));
+        curatorDatabase.create(deactivatedJobsPath());
     }
 
     /**
@@ -303,4 +307,27 @@ public class CuratorDatabaseClient {
     private Path defaultFlavorPath(ApplicationId applicationId) {
         return root.append("defaultFlavor").append(applicationId.serializedForm());
     }
+
+    public Set<String> readDeactivatedJobs() {
+        return curatorDatabase.getData(deactivatedJobsPath())
+                              .map(data -> stringSetSerializer.fromJson(data))
+                              .orElse(Collections.emptySet());
+    }
+
+    public void writeDeactivatedJobs(Set<String> deactivatedJobs) {
+        NestedTransaction transaction = new NestedTransaction();
+        CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+        curatorTransaction.add(CuratorOperations.setData(deactivatedJobsPath().getAbsolute(), 
+                                                         stringSetSerializer.toJson(deactivatedJobs)));
+        transaction.commit();
+    }
+    
+    public CuratorMutex lockDeactivatedJobs() {
+        return lock(deactivatedJobsPath(), defaultLockTimeout);
+    }
+
+    private Path deactivatedJobsPath() {
+        return root.append("deactivatedJobs");
+    }
+    
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java
new file mode 100644
index 00000000000..68a115a8ad5
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java
@@ -0,0 +1,43 @@
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Serialization of a set of strings to/from Json bytes using Slime.
+ * 
+ * The set is serialized as an array of string.
+ * 
+ * @author bratseth
+ */
+public class StringSetSerializer {
+
+    public byte[] toJson(Set<String> stringSet) {
+        try {
+            Slime slime = new Slime();
+            Cursor array = slime.setArray();
+            for (String element : stringSet)
+                array.addString(element);
+            return SlimeUtils.toJsonBytes(slime);
+        }
+        catch (IOException e) {
+            throw new RuntimeException("Serialization of a string set failed", e);
+        }
+        
+    }
+
+    public Set<String> fromJson(byte[] data) {
+        Inspector inspector = SlimeUtils.jsonToSlime(data).get();
+        Set<String> stringSet = new HashSet<>();
+        inspector.traverse((ArrayTraverser) (index, name) -> stringSet.add(name.asString()));
+        return stringSet;
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
index 868291b79dc..6aa8520b1f5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
@@ -136,7 +136,7 @@ public class FailedExpirerTest {
 
         // Failure times out
         clock.advance(Duration.ofDays(5));
-        new FailedExpirer(nodeRepository, new Zone(system, environment, RegionName.from("us-west-1")), clock, Duration.ofDays(4), new JobControl()).run();
+        new FailedExpirer(nodeRepository, new Zone(system, environment, RegionName.from("us-west-1")), clock, Duration.ofDays(4), new JobControl(nodeRepository.database())).run();
 
         return nodeRepository;
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
index bdd3af76e72..a07deff3543 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
@@ -52,7 +52,7 @@ public class InactiveAndFailedExpirerTest {
 
         // Inactive times out
         tester.advanceTime(Duration.ofMinutes(14));
-        new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new JobControl()).run();
+        new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database())).run();
         assertEquals(0, tester.nodeRepository().getNodes(Node.State.inactive).size());
         List<Node> dirty = tester.nodeRepository().getNodes(Node.State.dirty);
         assertEquals(2, dirty.size());
@@ -66,7 +66,7 @@ public class InactiveAndFailedExpirerTest {
 
         // Dirty times out for the other one
         tester.advanceTime(Duration.ofMinutes(14));
-        new DirtyExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new JobControl()).run();
+        new DirtyExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database())).run();
         assertEquals(0, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.dirty).size());
         List<Node> failed = tester.nodeRepository().getNodes(NodeType.tenant, Node.State.failed);
         assertEquals(1, failed.size());
@@ -96,7 +96,7 @@ public class InactiveAndFailedExpirerTest {
 
         // Inactive times out and node is moved to dirty
         tester.advanceTime(Duration.ofMinutes(14));
-        new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new JobControl()).run();
+        new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database())).run();
         List<Node> dirty = tester.nodeRepository().getNodes(Node.State.dirty);
         assertEquals(1, dirty.size());
 
@@ -138,12 +138,12 @@ public class InactiveAndFailedExpirerTest {
                                                             1)
                 )
         );
-        new RetiredExpirer(tester.nodeRepository(), deployer, tester.clock(), Duration.ofMinutes(10), new JobControl()).run();
+        new RetiredExpirer(tester.nodeRepository(), deployer, tester.clock(), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database())).run();
         assertEquals(1, tester.nodeRepository().getNodes(Node.State.inactive).size());
 
         // Inactive times out and one node is moved to parked
         tester.advanceTime(Duration.ofMinutes(11)); // Trigger InactiveExpirer
-        new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new JobControl()).run();
+        new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database())).run();
         assertEquals(1, tester.nodeRepository().getNodes(Node.State.parked).size());
     }
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java
new file mode 100644
index 00000000000..28d810bff43
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java
@@ -0,0 +1,51 @@
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.vespa.hosted.provision.NodeRepositoryTester;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ */
+public class JobControlTest {
+    
+    @Test
+    public void testJobControl() {
+        NodeRepositoryTester tester = new NodeRepositoryTester();
+        JobControl jobControl = new JobControl(tester.nodeRepository().database());
+        
+        assertTrue(jobControl.jobs().isEmpty());
+
+        String job1 = "Job1";
+        String job2 = "Job2";
+        
+        jobControl.started(job1);
+        jobControl.started(job2);
+        assertEquals(2, jobControl.jobs().size());
+        assertTrue(jobControl.jobs().contains(job1));
+        assertTrue(jobControl.jobs().contains(job2));
+        
+        assertTrue(jobControl.isActive(job1));
+        assertTrue(jobControl.isActive(job2));
+        
+        jobControl.setActive(job1, false);
+        assertFalse(jobControl.isActive(job1));
+        assertTrue(jobControl.isActive(job2));
+
+        jobControl.setActive(job2, false);
+        assertFalse(jobControl.isActive(job1));
+        assertFalse(jobControl.isActive(job2));
+
+        jobControl.setActive(job1, true);
+        assertTrue(jobControl.isActive(job1));
+        assertFalse(jobControl.isActive(job2));
+
+        jobControl.setActive(job2, true);
+        assertTrue(jobControl.isActive(job1));
+        assertTrue(jobControl.isActive(job2));
+    }
+    
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
index 9223412be9a..db6ba678be7 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
@@ -33,6 +33,8 @@ public class MaintenanceTester {
     public final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone,
                                                                      new MockNameResolver().mockAnyLookup());
 
+    public NodeRepository nodeRepository() { return nodeRepository; }
+    
     public void createReadyTenantNodes(int count) {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
index a958a5a3944..4e51138eceb 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
@@ -195,7 +195,7 @@ public class NodeFailTester {
     }
 
     public NodeFailer createFailer() {
-        return new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, downtimeLimitOneHour, clock, orchestrator, NodeFailer.ThrottlePolicy.hosted, new JobControl());
+        return new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, downtimeLimitOneHour, clock, orchestrator, NodeFailer.ThrottlePolicy.hosted, new JobControl(nodeRepository.database()));
     }
 
     public void allNodesMakeAConfigRequestExcept(Node ... deadNodeArray) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
index 3810dc2297b..3527d4ea04d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
@@ -25,7 +25,7 @@ public class NodeRebooterTest {
         // will be performed.
         tester.clock.advance(rebootInterval);
         
-        NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, rebootInterval, new JobControl());
+        NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, rebootInterval, new JobControl(tester.nodeRepository().database()));
 
         maintenanceIntervals(rebooter, tester, 1);
         assertEquals("All tenant nodes have reboot scheduled",
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
index d8e45f8d0e0..5ce876bf12c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
@@ -61,7 +61,7 @@ public class OperatorChangeApplicationMaintainerTest {
 
         // Create applications
         fixture.activate();
-        OperatorChangeApplicationMaintainer maintainer = new OperatorChangeApplicationMaintainer(fixture.deployer, nodeRepository, clock, Duration.ofMinutes(1), new JobControl());
+        OperatorChangeApplicationMaintainer maintainer = new OperatorChangeApplicationMaintainer(fixture.deployer, nodeRepository, clock, Duration.ofMinutes(1), new JobControl(nodeRepository.database()));
         
         clock.advance(Duration.ofMinutes(2));
         maintainer.maintain();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index 1a1c00947a7..844571acce5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -209,7 +209,7 @@ public class PeriodicApplicationMaintainerTest {
         
         TestablePeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval,
                                               Optional<List<Node>> overriddenNodesNeedingMaintenance) {
-            super(deployer, nodeRepository, interval, new JobControl());
+            super(deployer, nodeRepository, interval, new JobControl(nodeRepository.database()));
             this.overriddenNodesNeedingMaintenance = overriddenNodesNeedingMaintenance;
         }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
index a0d6558fdf9..f78dc031b0d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
@@ -59,7 +59,7 @@ public class ReservationExpirerTest {
 
         // Reservation times out
         clock.advance(Duration.ofMinutes(14)); // Reserved but not used time out
-        new ReservationExpirer(nodeRepository, clock, Duration.ofMinutes(10), new JobControl()).run();
+        new ReservationExpirer(nodeRepository, clock, Duration.ofMinutes(10), new JobControl(nodeRepository.database())).run();
 
         // Assert nothing is reserved
         assertEquals(0, nodeRepository.getNodes(NodeType.tenant, Node.State.reserved).size());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
index 0bb1fd64a5b..085f711058b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
@@ -71,7 +71,7 @@ public class RetiredExpirerTest {
         MockDeployer deployer =
             new MockDeployer(provisioner,
                              Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromNodeCount(wantedNodes, Optional.of("default")), 1)));
-        new RetiredExpirer(nodeRepository, deployer, clock, Duration.ofHours(12), new JobControl()).run();
+        new RetiredExpirer(nodeRepository, deployer, clock, Duration.ofHours(12), new JobControl(nodeRepository.database())).run();
         assertEquals(3, nodeRepository.getNodes(applicationId, Node.State.active).size());
         assertEquals(4, nodeRepository.getNodes(applicationId, Node.State.inactive).size());
         assertEquals(1, deployer.redeployments);
@@ -106,7 +106,7 @@ public class RetiredExpirerTest {
         MockDeployer deployer =
             new MockDeployer(provisioner,
                              Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromNodeCount(1, Optional.of("default")), 1)));
-        new RetiredExpirer(nodeRepository, deployer, clock, Duration.ofHours(12), new JobControl()).run();
+        new RetiredExpirer(nodeRepository, deployer, clock, Duration.ofHours(12), new JobControl(nodeRepository.database())).run();
         assertEquals(1, nodeRepository.getNodes(applicationId, Node.State.active).size());
         assertEquals(7, nodeRepository.getNodes(applicationId, Node.State.inactive).size());
         assertEquals(1, deployer.redeployments);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java
index 3631f6bf39c..54a6711c51d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java
@@ -25,7 +25,7 @@ public class ZooKeeperAccessMaintainerTest {
         NodeRepositoryTester tester = new NodeRepositoryTester();
         tester.curator().setConnectionSpec("server1:1234,server2:5678");
         ZooKeeperAccessMaintainer maintainer = new ZooKeeperAccessMaintainer(tester.nodeRepository(), 
-                                                                             tester.curator(), Duration.ofHours(1), new JobControl());
+                                                                             tester.curator(), Duration.ofHours(1), new JobControl(tester.nodeRepository().database()));
         assertTrue(ZooKeeperServer.getAllowedClientHostnames().isEmpty());
         maintainer.maintain();
         assertTrue("We don't restrict to only config servers", ZooKeeperServer.getAllowedClientHostnames().isEmpty());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java
index 158feea0d49..fdfd4bdd8bf 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java
@@ -52,7 +52,7 @@ public class MetricsReporterTest {
         expectedMetrics.put("hostedVespa.failedHosts", 0);
 
         TestMetric metric = new TestMetric();
-        MetricsReporter metricsReporter = new MetricsReporter(nodeRepository, metric, Duration.ofMinutes(1), new JobControl());
+        MetricsReporter metricsReporter = new MetricsReporter(nodeRepository, metric, Duration.ofMinutes(1), new JobControl(nodeRepository.database()));
         metricsReporter.maintain();
 
         assertEquals(expectedMetrics, metric.values);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
index 350aea689cd..b4f17cc003c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
@@ -136,7 +136,7 @@ public class MultigroupProvisioningTest {
                              Collections.singletonMap(application1, 
                                                       new MockDeployer.ApplicationContext(application1, cluster(), 
                                                                                           Capacity.fromNodeCount(8, Optional.of("large")), 1)));
-        new RetiredExpirer(tester.nodeRepository(), deployer, tester.clock(), Duration.ofHours(12), new JobControl()).run();
+        new RetiredExpirer(tester.nodeRepository(), deployer, tester.clock(), Duration.ofHours(12), new JobControl(tester.nodeRepository().database())).run();
 
         assertEquals(8, tester.getNodes(application1, Node.State.inactive).flavor("small").size());
         deploy(application1, 8, 8, "large", tester);