Merge pull request #7426 from vespa-engine/freva/reboot-container

NodeAdmin: Support container reboot

26 files changed, 256 insertions, 606 deletions

diff --git a/node-admin/pom.xml b/node-admin/pom.xml
index 64958554f53..476902e400c 100644
--- a/node-admin/pom.xml
+++ b/node-admin/pom.xml
@@ -91,6 +91,7 @@
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
+            <version>2.23.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java
index 9a94231437d..91ff159ac41 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java
@@ -130,6 +130,6 @@ public class NodeAttributes {
                         wantToDeprovision.map(depr -> "wantToDeprovision=" + depr))
                 .filter(Optional::isPresent)
                 .map(Optional::get)
-                .collect(Collectors.joining(", ", "NodeAttributes{", "}"));
+                .collect(Collectors.joining(", ", "{", "}"));
     }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index ef137c55ffb..a220557ca9c 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -33,6 +33,7 @@ import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -236,26 +237,33 @@ public class NodeAgentImpl implements NodeAgent {
     }
 
     private void updateNodeRepoWithCurrentAttributes(final NodeSpec node) {
-        final NodeAttributes currentNodeAttributes = new NodeAttributes()
-                .withRestartGeneration(node.getCurrentRestartGeneration())
-                .withRebootGeneration(node.getCurrentRebootGeneration())
-                .withDockerImage(node.getCurrentDockerImage().orElse(new DockerImage("")));
-
-        final NodeAttributes wantedNodeAttributes = new NodeAttributes()
-                .withRestartGeneration(node.getWantedRestartGeneration())
-                // update reboot gen with wanted gen if set, we ignore reboot for Docker nodes but
-                // want the two to be equal in node repo
-                .withRebootGeneration(node.getWantedRebootGeneration())
-                .withDockerImage(node.getWantedDockerImage().filter(n -> containerState == UNKNOWN).orElse(new DockerImage("")));
-
-        publishStateToNodeRepoIfChanged(currentNodeAttributes, wantedNodeAttributes);
+        final NodeAttributes currentNodeAttributes = new NodeAttributes();
+        final NodeAttributes newNodeAttributes = new NodeAttributes();
+
+        if (!Objects.equals(node.getCurrentRestartGeneration(), node.getWantedRestartGeneration())) {
+            currentNodeAttributes.withRestartGeneration(node.getCurrentRestartGeneration());
+            newNodeAttributes.withRestartGeneration(node.getWantedRestartGeneration());
+        }
+
+        if (!Objects.equals(node.getCurrentRebootGeneration(), node.getWantedRebootGeneration())) {
+            currentNodeAttributes.withRebootGeneration(node.getCurrentRebootGeneration());
+            newNodeAttributes.withRebootGeneration(node.getWantedRebootGeneration());
+        }
+
+        Optional<DockerImage> actualDockerImage = node.getWantedDockerImage().filter(n -> containerState == UNKNOWN);
+        if (!Objects.equals(node.getCurrentDockerImage(), actualDockerImage)) {
+            currentNodeAttributes.withDockerImage(node.getCurrentDockerImage().orElse(new DockerImage("")));
+            newNodeAttributes.withDockerImage(actualDockerImage.orElse(new DockerImage("")));
+        }
+
+        publishStateToNodeRepoIfChanged(currentNodeAttributes, newNodeAttributes);
     }
 
-    private void publishStateToNodeRepoIfChanged(NodeAttributes currentAttributes, NodeAttributes wantedAttributes) {
-        if (!currentAttributes.equals(wantedAttributes)) {
+    private void publishStateToNodeRepoIfChanged(NodeAttributes currentAttributes, NodeAttributes newAttributes) {
+        if (!currentAttributes.equals(newAttributes)) {
             context.log(logger, "Publishing new set of attributes to node repo: %s -> %s",
-                    currentAttributes, wantedAttributes);
-            nodeRepository.updateNodeAttributes(context.hostname().value(), wantedAttributes);
+                    currentAttributes, newAttributes);
+            nodeRepository.updateNodeAttributes(context.hostname().value(), newAttributes);
         }
     }
 
@@ -285,8 +293,8 @@ public class NodeAgentImpl implements NodeAgent {
     private Optional<String> shouldRestartServices(NodeSpec node) {
         if (!node.getWantedRestartGeneration().isPresent()) return Optional.empty();
 
-        if (!node.getCurrentRestartGeneration().isPresent() ||
-                node.getCurrentRestartGeneration().get() < node.getWantedRestartGeneration().get()) {
+        // Restart generation is only optional because it does not exist for unallocated nodes
+        if (node.getCurrentRestartGeneration().get() < node.getWantedRestartGeneration().get()) {
             return Optional.of("Restart requested - wanted restart generation has been bumped: "
                     + node.getCurrentRestartGeneration().get() + " -> " + node.getWantedRestartGeneration().get());
         }
@@ -350,6 +358,11 @@ public class NodeAgentImpl implements NodeAgent {
                     wantedContainerResources + ", actual: " + existingContainer.resources);
         }
 
+        if (node.getCurrentRebootGeneration() < node.getWantedRebootGeneration()) {
+            return Optional.of(String.format("Container reboot wanted. Current: %d, Wanted: %d",
+                    node.getCurrentRebootGeneration(), node.getWantedRebootGeneration()));
+        }
+
         if (containerState == STARTING) return Optional.of("Container failed to start");
         return Optional.empty();
     }
@@ -466,7 +479,6 @@ public class NodeAgentImpl implements NodeAgent {
                 new IllegalStateException(String.format("Node '%s' missing from node repository", context.hostname())));
         expectNodeNotInNodeRepo = false;
 
-
         Optional<Container> container = getContainer();
         if (!node.equals(lastNode)) {
             // Every time the node spec changes, we should clear the metrics for this container as the dimensions
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java
index da7b1e6a00c..2cff62afad6 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java
@@ -26,7 +26,7 @@ import static org.hamcrest.junit.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java
index 01aaa385d85..2604aa05367 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java
@@ -8,7 +8,7 @@ import org.junit.Test;
 import java.net.ConnectException;
 
 import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
index 2a4bf6bf488..6e8cfce6c37 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
@@ -28,9 +28,8 @@ import java.util.OptionalLong;
 import static org.hamcrest.core.Is.is;
 import static org.junit.Assert.assertThat;
 import static org.mockito.AdditionalMatchers.aryEq;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyVararg;
-import static org.mockito.Matchers.eq;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -50,7 +49,7 @@ public class DockerOperationsImplTest {
         final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build();
         final ProcessResult actualResult = new ProcessResult(0, "output", "errors");
 
-        when(docker.executeInContainerAsUser(any(), any(), any(), anyVararg()))
+        when(docker.executeInContainerAsUser(any(), any(), any(), any()))
                 .thenReturn(actualResult); // output from node program
 
         ProcessResult result = dockerOperations.executeNodeCtlInContainer(context, "start");
@@ -71,7 +70,7 @@ public class DockerOperationsImplTest {
         final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build();
         final ProcessResult actualResult = new ProcessResult(3, "output", "errors");
 
-        when(docker.executeInContainerAsUser(any(), any(), any(), anyVararg()))
+        when(docker.executeInContainerAsUser(any(), any(), any(), any()))
                 .thenReturn(actualResult); // output from node program
 
         dockerOperations.executeNodeCtlInContainer(context, "start");
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/CallOrderVerifier.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/CallOrderVerifier.java
deleted file mode 100644
index df351d4cc2e..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/CallOrderVerifier.java
+++ /dev/null
@@ -1,114 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Takes in strings representing function calls with their parameters and allows to check whether a subset of calls
- * occurred in a specific order. For example, by calling {@link CallOrderVerifier#add(String)}
- * with A, B, C, D, E, D, A, F, D and G, then {@link CallOrderVerifier#verifyInOrder(String...)} with
- * A, B, D => true,
- * A, D, A => true,
- * C, D, F => true,
- * B, D, A => true
- * B, F, D, A => false,
- * C, B => false
- *
- * @author freva
- */
-public class CallOrderVerifier {
-    private static final int waitForCallOrderTimeout = 600; //ms
-
-    private final LinkedList<String> callOrder = new LinkedList<>();
-    private final Object monitor = new Object();
-
-    public void add(String call) {
-        synchronized (monitor) {
-            callOrder.add(call);
-        }
-    }
-
-    public void assertInOrder(String... functionCalls) {
-        assertInOrder(waitForCallOrderTimeout, functionCalls);
-    }
-
-    public void assertInOrder(long timeout, String... functionCalls) {
-        assertInOrderWithAssertMessage(timeout, "", functionCalls);
-    }
-
-    public void assertInOrderWithAssertMessage(String assertMessage, String... functionCalls) {
-        assertInOrderWithAssertMessage(waitForCallOrderTimeout, assertMessage, functionCalls);
-    }
-
-    public void assertInOrderWithAssertMessage(long timeout, String assertMessage, String... functionCalls) {
-        boolean inOrder = verifyInOrder(timeout, functionCalls);
-        if ( ! inOrder && ! assertMessage.isEmpty())
-            System.err.println(assertMessage);
-        assertTrue(toString(), inOrder);
-    }
-
-    /**
-     * Checks if list of function calls occur in order given within a timeout
-     * @param timeout Max number of milliseconds to check for if function calls occur in order
-     * @param functionCalls The expected order of function calls
-     * @return true if the actual order of calls was equal to the order provided within timeout, false otherwise.
-     */
-    private boolean verifyInOrder(long timeout, String... functionCalls) {
-        final long startTime = System.currentTimeMillis();
-        while (System.currentTimeMillis() - startTime < timeout) {
-            if (verifyInOrder(functionCalls)) {
-                return true;
-            }
-            try {
-                Thread.sleep(10);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        }
-
-        return false;
-    }
-
-    private boolean verifyInOrder(String... functionCalls) {
-        int pos = 0;
-        synchronized (monitor) {
-            for (String functionCall : functionCalls) {
-                int temp = indexOf(callOrder.listIterator(pos), functionCall);
-                if (temp == -1) {
-                    System.out.println("Function call '" + functionCall +
-                            "' never made at position " + pos);
-                    return false;
-                }
-                pos += temp;
-            }
-        }
-
-        return true;
-    }
-
-    /**
-     * Finds the first index of needle in haystack after a given position.
-     * @param iter Iterator to search in
-     * @param search Element to find in iterator
-     * @return Index of the next search in  after startPos, -1 if not found
-     */
-    private int indexOf(Iterator<String> iter, String search) {
-        for (int i = 0; iter.hasNext(); i++) {
-            if (search.equals(iter.next())) {
-                return i;
-            }
-        }
-
-        return -1;
-    }
-
-    @Override
-    public String toString() {
-        synchronized (monitor) {
-            return callOrder.toString();
-        }
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
index d7e0fe3d8e5..b3bf2282988 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
@@ -3,22 +3,32 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.provision.Node;
 import org.junit.Test;
 
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
 /**
  * @author freva
  */
 public class DockerFailTest {
 
     @Test
-    public void dockerFailTest() throws Exception {
-        try (DockerTester dockerTester = new DockerTester()) {
-            NodeSpec nodeSpec = new NodeSpec.Builder()
-                    .hostname("host1.test.yahoo.com")
-                    .wantedDockerImage(new DockerImage("dockerImage"))
+    public void dockerFailTest() {
+        try (DockerTester tester = new DockerTester()) {
+            final DockerImage dockerImage = new DockerImage("dockerImage");
+            final ContainerName containerName = new ContainerName("host1");
+            final String hostname = "host1.test.yahoo.com";
+            tester.addChildNodeRepositoryNode(new NodeSpec.Builder()
+                    .hostname(hostname)
+                    .wantedDockerImage(dockerImage)
+                    .currentDockerImage(dockerImage)
                     .state(Node.State.active)
                     .nodeType(NodeType.tenant)
                     .flavor("docker")
@@ -27,24 +37,22 @@ public class DockerFailTest {
                     .minCpuCores(1)
                     .minMainMemoryAvailableGb(1)
                     .minDiskAvailableGb(1)
-                    .build();
-            dockerTester.addChildNodeRepositoryNode(nodeSpec);
+                    .build());
 
-            // Wait for node admin to be notified with node repo state and the docker container has been started
-            while (dockerTester.nodeAdmin.getNumberOfNodeAgents() == 0) {
-                Thread.sleep(10);
-            }
+            tester.inOrder(tester.docker).createContainerCommand(
+                    eq(dockerImage), eq(ContainerResources.from(1, 1)), eq(containerName), eq(hostname));
+            tester.inOrder(tester.docker).executeInContainerAsUser(
+                    eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume"));
 
-            dockerTester.callOrderVerifier.assertInOrder(1200,
-                    "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+            tester.docker.deleteContainer(new ContainerName("host1"));
 
-            dockerTester.dockerMock.deleteContainer(new ContainerName("host1"));
+            tester.inOrder(tester.docker).deleteContainer(eq(containerName));
+            tester.inOrder(tester.docker).createContainerCommand(
+                    eq(dockerImage), eq(ContainerResources.from(1, 1)), eq(containerName), eq(hostname));
+            tester.inOrder(tester.docker).executeInContainerAsUser(
+                    eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume"));
 
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "deleteContainer with ContainerName { name=host1 }",
-                    "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+            verify(tester.nodeRepository, never()).updateNodeAttributes(any(), any());
         }
     }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
index 26843b119de..75bdaed60cf 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
@@ -13,7 +13,6 @@ import java.net.InetAddress;
 import java.nio.file.Path;
 import java.time.Duration;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -27,13 +26,8 @@ import java.util.OptionalLong;
  */
 public class DockerMock implements Docker {
     private final Map<ContainerName, Container> containersByContainerName = new HashMap<>();
-    public final CallOrderVerifier callOrderVerifier;
     private static final Object monitor = new Object();
 
-    public DockerMock(CallOrderVerifier callOrderVerifier) {
-        this.callOrderVerifier = callOrderVerifier;
-    }
-
     @Override
     public CreateContainerCommand createContainerCommand(
             DockerImage dockerImage,
@@ -41,8 +35,6 @@ public class DockerMock implements Docker {
             ContainerName containerName,
             String hostName) {
         synchronized (monitor) {
-            callOrderVerifier.add("createContainerCommand with " + dockerImage +
-                    ", HostName: " + hostName + ", " + containerName);
             containersByContainerName.put(
                     containerName, new Container(hostName, dockerImage, containerResources, containerName, Container.State.RUNNING, 2));
         }
@@ -64,15 +56,12 @@ public class DockerMock implements Docker {
 
     @Override
     public void startContainer(ContainerName containerName) {
-        synchronized (monitor) {
-            callOrderVerifier.add("startContainer with " + containerName);
-        }
+
     }
 
     @Override
     public void stopContainer(ContainerName containerName) {
         synchronized (monitor) {
-            callOrderVerifier.add("stopContainer with " + containerName);
             Container container = containersByContainerName.get(containerName);
             containersByContainerName.put(containerName,
                             new Container(container.hostname, container.image, container.resources, container.name, Container.State.EXITED, 0));
@@ -82,7 +71,6 @@ public class DockerMock implements Docker {
     @Override
     public void deleteContainer(ContainerName containerName) {
         synchronized (monitor) {
-            callOrderVerifier.add("deleteContainer with " + containerName);
             containersByContainerName.remove(containerName);
         }
     }
@@ -97,7 +85,6 @@ public class DockerMock implements Docker {
     @Override
     public boolean pullImageAsyncIfNeeded(DockerImage image) {
         synchronized (monitor) {
-            callOrderVerifier.add("pullImageAsyncIfNeeded with " + image);
             return false;
         }
     }
@@ -109,9 +96,6 @@ public class DockerMock implements Docker {
 
     @Override
     public ProcessResult executeInContainerAsUser(ContainerName containerName, String user, OptionalLong timeout, String... args) {
-        synchronized (monitor) {
-            callOrderVerifier.add("executeInContainer " + containerName.asString() + " as " + user + ", args: " + Arrays.toString(args));
-        }
         return new ProcessResult(0, null, "");
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
index aa40d5d805b..6ed51657d6e 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
@@ -9,8 +9,10 @@ import com.yahoo.system.ProcessExecuter;
 import com.yahoo.vespa.hosted.dockerapi.Docker;
 import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl;
+import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent;
@@ -19,6 +21,8 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl;
 import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.test.file.TestFileSystem;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
 
 import java.nio.file.FileSystem;
 import java.nio.file.Path;
@@ -30,8 +34,10 @@ import java.util.Optional;
 import java.util.function.Function;
 
 import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck;
-import static org.mockito.Matchers.any;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
 import static org.mockito.Mockito.when;
 
 /**
@@ -45,16 +51,19 @@ public class DockerTester implements AutoCloseable {
     static final String NODE_PROGRAM = PATH_TO_VESPA_HOME.resolve("bin/vespa-nodectl").toString();
     static final HostName HOST_HOSTNAME = HostName.from("host.test.yahoo.com");
 
-    final CallOrderVerifier callOrderVerifier = new CallOrderVerifier();
-    final Docker dockerMock = new DockerMock(callOrderVerifier);
-    final NodeRepoMock nodeRepositoryMock = new NodeRepoMock(callOrderVerifier);
+    final Docker docker = spy(new DockerMock());
+    final NodeRepoMock nodeRepository = spy(new NodeRepoMock());
+    final Orchestrator orchestrator = mock(Orchestrator.class);
+    final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class);
+    final InOrder inOrder = Mockito.inOrder(docker, nodeRepository, orchestrator, storageMaintainer);
+
     final NodeAdminStateUpdaterImpl nodeAdminStateUpdater;
     final NodeAdminImpl nodeAdmin;
-    private final OrchestratorMock orchestratorMock = new OrchestratorMock(callOrderVerifier);
-    private final FileSystem fileSystem = TestFileSystem.create();
 
 
     DockerTester() {
+        when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.empty());
+
         IPAddressesMock ipAddresses = new IPAddressesMock();
         ipAddresses.addAddress(HOST_HOSTNAME.value(), "1.1.1.1");
         ipAddresses.addAddress(HOST_HOSTNAME.value(), "f000::");
@@ -71,25 +80,25 @@ public class DockerTester implements AutoCloseable {
                 .wantedRestartGeneration(1L)
                 .currentRestartGeneration(1L)
                 .build();
-        nodeRepositoryMock.updateNodeRepositoryNode(hostSpec);
+        nodeRepository.updateNodeRepositoryNode(hostSpec);
 
         Clock clock = Clock.systemUTC();
-        DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, processExecuter, node -> "", Collections.emptyList(), ipAddresses);
-        StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, callOrderVerifier);
+        FileSystem fileSystem = TestFileSystem.create();
+        DockerOperations dockerOperations = new DockerOperationsImpl(docker, processExecuter, node -> "", Collections.emptyList(), ipAddresses);
 
         MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
         Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(
-                new NodeAgentContextImpl.Builder(hostName).fileSystem(fileSystem).build(), nodeRepositoryMock,
-                orchestratorMock, dockerOperations, storageMaintainer, clock, NODE_AGENT_SCAN_INTERVAL, Optional.empty(), Optional.empty());
+                new NodeAgentContextImpl.Builder(hostName).fileSystem(fileSystem).build(), nodeRepository,
+                orchestrator, dockerOperations, storageMaintainer, clock, NODE_AGENT_SCAN_INTERVAL, Optional.empty(), Optional.empty());
         nodeAdmin = new NodeAdminImpl(nodeAgentFactory, Optional.empty(), mr, Clock.systemUTC());
-        nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock,
+        nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepository, orchestrator,
                 nodeAdmin, HOST_HOSTNAME, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL);
         nodeAdminStateUpdater.start();
     }
 
     /** Adds a node to node-repository mock that is running on this host */
     void addChildNodeRepositoryNode(NodeSpec nodeSpec) {
-        nodeRepositoryMock.updateNodeRepositoryNode(new NodeSpec.Builder(nodeSpec)
+        nodeRepository.updateNodeRepositoryNode(new NodeSpec.Builder(nodeSpec)
                 .parentHostname(HOST_HOSTNAME.value())
                 .build());
     }
@@ -98,4 +107,8 @@ public class DockerTester implements AutoCloseable {
     public void close() {
         nodeAdminStateUpdater.stop();
     }
+
+    public <T> T inOrder(T t) {
+        return inOrder.verify(t, timeout(1000));
+    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
index e130cd0d475..48fdc564aac 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
@@ -3,24 +3,30 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.provision.Node;
 import org.junit.Test;
 
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+
 /**
  * @author freva
  */
 public class MultiDockerTest {
 
     @Test
-    public void test() throws InterruptedException {
-        try (DockerTester dockerTester = new DockerTester()) {
-            addAndWaitForNode(dockerTester, "host1.test.yahoo.com", new DockerImage("image1"));
+    public void test() {
+        try (DockerTester tester = new DockerTester()) {
+            addAndWaitForNode(tester, "host1.test.yahoo.com", new DockerImage("image1"));
             NodeSpec nodeSpec2 = addAndWaitForNode(
-                    dockerTester, "host2.test.yahoo.com", new DockerImage("image2"));
+                    tester, "host2.test.yahoo.com", new DockerImage("image2"));
 
-            dockerTester.addChildNodeRepositoryNode(
+            tester.addChildNodeRepositoryNode(
                     new NodeSpec.Builder(nodeSpec2)
                             .state(Node.State.dirty)
                             .minCpuCores(1)
@@ -28,66 +34,37 @@ public class MultiDockerTest {
                             .minDiskAvailableGb(1)
                             .build());
 
-            // Wait until it is marked ready
-            while (dockerTester.nodeRepositoryMock.getOptionalNode(nodeSpec2.getHostname())
-                    .filter(node -> node.getState() != Node.State.ready).isPresent()) {
-                Thread.sleep(10);
-            }
-
-            addAndWaitForNode(dockerTester, "host3.test.yahoo.com", new DockerImage("image1"));
-
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "createContainerCommand with DockerImage { imageId=image1 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]",
-
-                    "createContainerCommand with DockerImage { imageId=image2 }, HostName: host2.test.yahoo.com, ContainerName { name=host2 }",
-                    "executeInContainer host2 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]",
-
-                    "stopContainer with ContainerName { name=host2 }",
-                    "deleteContainer with ContainerName { name=host2 }",
+            tester.inOrder(tester.docker).deleteContainer(eq(new ContainerName("host2")));
+            tester.inOrder(tester.storageMaintainer).archiveNodeStorage(
+                    argThat(context -> context.containerName().equals(new ContainerName("host2"))));
+            tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.getHostname()), eq(Node.State.ready));
 
-                    "createContainerCommand with DockerImage { imageId=image1 }, HostName: host3.test.yahoo.com, ContainerName { name=host3 }",
-                    "executeInContainer host3 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
-
-            dockerTester.callOrderVerifier.assertInOrderWithAssertMessage(
-                    "Maintainer did not receive call to delete application storage",
-                    "deleteContainer with ContainerName { name=host2 }",
-                    "DeleteContainerStorage with ContainerName { name=host2 }");
-
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1}",
-                    "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2}",
-                    "setNodeState host2.test.yahoo.com to ready",
-                    "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1}");
+            addAndWaitForNode(tester, "host3.test.yahoo.com", new DockerImage("image1"));
         }
     }
 
-    private NodeSpec addAndWaitForNode(DockerTester tester, String hostName, DockerImage dockerImage) throws InterruptedException {
+    private NodeSpec addAndWaitForNode(DockerTester tester, String hostName, DockerImage dockerImage) {
         NodeSpec nodeSpec = new NodeSpec.Builder()
                 .hostname(hostName)
                 .wantedDockerImage(dockerImage)
-                .wantedVespaVersion("1.2.3")
                 .state(Node.State.active)
                 .nodeType(NodeType.tenant)
                 .flavor("docker")
                 .wantedRestartGeneration(1L)
                 .currentRestartGeneration(1L)
-                .minCpuCores(1)
-                .minMainMemoryAvailableGb(1)
+                .minCpuCores(2)
+                .minMainMemoryAvailableGb(4)
                 .minDiskAvailableGb(1)
                 .build();
 
         tester.addChildNodeRepositoryNode(nodeSpec);
 
-        // Wait for node admin to be notified with node repo state and the docker container has been started
-        while (tester.nodeAdmin.getNumberOfNodeAgents() + 1 != tester.nodeRepositoryMock.getNumberOfContainerSpecs()) {
-            Thread.sleep(10);
-        }
-
         ContainerName containerName = ContainerName.fromHostname(hostName);
-        tester.callOrderVerifier.assertInOrder(
-                "createContainerCommand with " + dockerImage + ", HostName: " + hostName + ", " + containerName,
-                "executeInContainer " + containerName.asString() + " as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+        tester.inOrder(tester.docker).createContainerCommand(
+                eq(dockerImage), eq(ContainerResources.from(2, 4)), eq(containerName), eq(hostName));
+        tester.inOrder(tester.docker).executeInContainerAsUser(
+                eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume"));
+        tester.inOrder(tester.nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withDockerImage(dockerImage)));
 
         return nodeSpec;
     }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
index d25d79ab457..e8b423e73ac 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttribu
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
 import com.yahoo.vespa.hosted.provision.Node;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -23,13 +24,6 @@ public class NodeRepoMock implements NodeRepository {
     private static final Object monitor = new Object();
 
     private final Map<String, NodeSpec> nodeRepositoryNodesByHostname = new HashMap<>();
-    private final Map<String, Acl> acls = new HashMap<>();
-
-    private final CallOrderVerifier callOrderVerifier;
-
-    public NodeRepoMock(CallOrderVerifier callOrderVerifier) {
-        this.callOrderVerifier = callOrderVerifier;
-    }
 
     @Override
     public void addNodes(List<AddNode> nodes) { }
@@ -52,15 +46,15 @@ public class NodeRepoMock implements NodeRepository {
 
     @Override
     public Map<String, Acl> getAcls(String hostname) {
-        synchronized (monitor) {
-            return acls;
-        }
+        return Collections.emptyMap();
     }
 
     @Override
     public void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes) {
         synchronized (monitor) {
-            callOrderVerifier.add("updateNodeAttributes with HostName: " + hostName + ", " + nodeAttributes);
+            updateNodeRepositoryNode(new NodeSpec.Builder(getNode(hostName))
+                    .updateFromNodeAttributes(nodeAttributes)
+                    .build());
         }
     }
 
@@ -70,7 +64,6 @@ public class NodeRepoMock implements NodeRepository {
             updateNodeRepositoryNode(new NodeSpec.Builder(getNode(hostName))
                     .state(nodeState)
                     .build());
-            callOrderVerifier.add("setNodeState " + hostName + " to " + nodeState);
         }
     }
 
@@ -79,13 +72,9 @@ public class NodeRepoMock implements NodeRepository {
 
     }
 
-    public void updateNodeRepositoryNode(NodeSpec nodeSpec) {
-        nodeRepositoryNodesByHostname.put(nodeSpec.getHostname(), nodeSpec);
-    }
-
-    public int getNumberOfContainerSpecs() {
+    void updateNodeRepositoryNode(NodeSpec nodeSpec) {
         synchronized (monitor) {
-            return nodeRepositoryNodesByHostname.size();
+            nodeRepositoryNodesByHostname.put(nodeSpec.getHostname(), nodeSpec);
         }
     }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
deleted file mode 100644
index da743c40e8b..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
-
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.dockerapi.DockerImage;
-import com.yahoo.vespa.hosted.provision.Node;
-import org.junit.Test;
-
-/**
- * Test NodeState transitions in NodeRepository
- *
- * @author freva
- */
-public class NodeStateTest {
-    private final NodeSpec initialNodeSpec = new NodeSpec.Builder()
-            .hostname("host1.test.yahoo.com")
-            .wantedDockerImage(new DockerImage("dockerImage"))
-            .state(Node.State.active)
-            .nodeType(NodeType.tenant)
-            .flavor("docker")
-            .wantedRestartGeneration(1L)
-            .currentRestartGeneration(1L)
-            .minCpuCores(1)
-            .minMainMemoryAvailableGb(1)
-            .minDiskAvailableGb(1)
-            .build();
-
-    private void setup(DockerTester tester) throws InterruptedException {
-        tester.addChildNodeRepositoryNode(initialNodeSpec);
-
-        // Wait for node admin to be notified with node repo state and the docker container has been started
-        while (tester.nodeAdmin.getNumberOfNodeAgents() == 0) {
-            Thread.sleep(10);
-        }
-
-        tester.callOrderVerifier.assertInOrder(
-                "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
-    }
-
-
-    @Test
-    public void activeToDirty() throws InterruptedException {
-        try (DockerTester dockerTester = new DockerTester()) {
-            setup(dockerTester);
-            // Change node state to dirty
-            dockerTester.addChildNodeRepositoryNode(new NodeSpec.Builder(initialNodeSpec)
-                    .state(Node.State.dirty)
-                    .minCpuCores(1)
-                    .minMainMemoryAvailableGb(1)
-                    .minDiskAvailableGb(1)
-                    .build());
-
-            // Wait until it is marked ready
-            while (dockerTester.nodeRepositoryMock.getOptionalNode(initialNodeSpec.getHostname())
-                    .filter(node -> node.getState() != Node.State.ready).isPresent()) {
-                Thread.sleep(10);
-            }
-
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", stop]",
-                    "stopContainer with ContainerName { name=host1 }",
-                    "deleteContainer with ContainerName { name=host1 }");
-        }
-    }
-
-    @Test
-    public void activeToInactiveToActive() throws InterruptedException {
-
-        try (DockerTester dockerTester = new DockerTester()) {
-            setup(dockerTester);
-
-            DockerImage newDockerImage = new DockerImage("newDockerImage");
-
-            // Change node state to inactive and change the wanted docker image
-            dockerTester.addChildNodeRepositoryNode(new NodeSpec.Builder(initialNodeSpec)
-                    .wantedDockerImage(newDockerImage)
-                    .state(Node.State.inactive)
-                    .minCpuCores(1)
-                    .minMainMemoryAvailableGb(1)
-                    .minDiskAvailableGb(1)
-                    .build());
-
-            dockerTester.callOrderVerifier.assertInOrderWithAssertMessage(
-                    "Node set to inactive, but no stop/delete call received",
-                    "stopContainer with ContainerName { name=host1 }",
-                    "deleteContainer with ContainerName { name=host1 }");
-
-
-            // Change node state to active
-            dockerTester.addChildNodeRepositoryNode(new NodeSpec.Builder(initialNodeSpec)
-                    .wantedDockerImage(newDockerImage)
-                    .state(Node.State.active)
-                    .minCpuCores(1)
-                    .minMainMemoryAvailableGb(1)
-                    .minDiskAvailableGb(1)
-                    .build());
-
-            // Check that the container is started again after the delete call
-            dockerTester.callOrderVerifier.assertInOrderWithAssertMessage(
-                    "Node not started again after being put to active state",
-                    "deleteContainer with ContainerName { name=host1 }",
-                    "createContainerCommand with DockerImage { imageId=newDockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
-        }
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java
deleted file mode 100644
index 469022cec56..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
-
-import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
-
-import java.util.List;
-
-/**
- * Mock with some simple logic
- *
- * @author dybis
- */
-public class OrchestratorMock implements Orchestrator {
-    private final CallOrderVerifier callOrderVerifier;
-
-    OrchestratorMock(CallOrderVerifier callOrderVerifier) {
-        this.callOrderVerifier = callOrderVerifier;
-    }
-
-    @Override
-    public void suspend(String hostName) {
-        callOrderVerifier.add("Suspend for " + hostName);
-    }
-
-    @Override
-    public void resume(String hostName) {
-        callOrderVerifier.add("Resume for " + hostName);
-    }
-
-    @Override
-    public void suspend(String parentHostName, List<String> hostNames) {
-        callOrderVerifier.add("Suspend with parent: " + parentHostName + " and hostnames: " + hostNames);
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
index 778443295bf..75fefde427d 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
@@ -2,16 +2,24 @@
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin;
 import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater;
-import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl;
 import com.yahoo.vespa.hosted.provision.Node;
-import org.junit.Ignore;
 import org.junit.Test;
 
+import java.util.Arrays;
+import java.util.OptionalLong;
+
+import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.HOST_HOSTNAME;
+import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.NODE_PROGRAM;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
 
 /**
  * Tests rebooting of Docker host
@@ -20,47 +28,33 @@ import static org.junit.Assert.assertTrue;
  */
 public class RebootTest {
 
-    @Test
-    @Ignore
-    public void test() throws InterruptedException {
-        try (DockerTester dockerTester = new DockerTester()) {
-
-            dockerTester.addChildNodeRepositoryNode(createNodeRepositoryNode());
-
-            // Wait for node admin to be notified with node repo state and the docker container has been started
-            while (dockerTester.nodeAdmin.getNumberOfNodeAgents() == 0) {
-                Thread.sleep(10);
-            }
+    private final String hostname = "host1.test.yahoo.com";
+    private final DockerImage dockerImage = new DockerImage("dockerImage");
 
-            // Check that the container is started and NodeRepo has received the PATCH update
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=null,  dockerImage=dockerImage, vespaVersion='null'}");
-
-            NodeAdminStateUpdaterImpl updater = dockerTester.nodeAdminStateUpdater;
-//            assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED),
-//                       is(Optional.of("Not all node agents are frozen.")));
-
-            updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED);
+    @Test
+    public void test() {
+        try (DockerTester tester = new DockerTester()) {
+            tester.addChildNodeRepositoryNode(createNodeRepositoryNode());
 
-            NodeAdmin nodeAdmin = dockerTester.nodeAdmin;
-            // Wait for node admin to be frozen
-            while ( ! nodeAdmin.isFrozen()) {
-                System.out.println("Node admin not frozen yet");
-                Thread.sleep(10);
-            }
+            tester.inOrder(tester.docker).createContainerCommand(
+                    eq(dockerImage), eq(ContainerResources.from(0, 0)), eq(new ContainerName("host1")), eq(hostname));
 
-            assertTrue(nodeAdmin.setFrozen(false));
+            try {
+                tester.nodeAdminStateUpdater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED);
+            } catch (RuntimeException ignored) { }
 
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "executeInContainer with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", stop]");
+            tester.inOrder(tester.orchestrator).suspend(
+                    eq(HOST_HOSTNAME.value()), eq(Arrays.asList(hostname, HOST_HOSTNAME.value())));
+            tester.inOrder(tester.docker).executeInContainerAsUser(
+                    eq(new ContainerName("host1")), eq("root"), eq(OptionalLong.empty()), eq(NODE_PROGRAM), eq("stop"));
+            assertTrue(tester.nodeAdmin.setFrozen(true));
         }
     }
 
     private NodeSpec createNodeRepositoryNode() {
         return new NodeSpec.Builder()
-                .hostname("host1.test.yahoo.com")
-                .wantedDockerImage(new DockerImage("dockerImage"))
+                .hostname(hostname)
+                .wantedDockerImage(dockerImage)
                 .state(Node.State.active)
                 .nodeType(NodeType.tenant)
                 .flavor("docker")
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
index 2e290e014c2..2b8727529f2 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
@@ -2,11 +2,19 @@
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.provision.Node;
 import org.junit.Test;
 
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.NODE_PROGRAM;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+
 /**
  * Tests that different wanted and current restart generation leads to execution of restart command
  *
@@ -15,46 +23,35 @@ import org.junit.Test;
 public class RestartTest {
 
     @Test
-    public void test() throws InterruptedException {
-        try (DockerTester dockerTester = new DockerTester()) {
-
-            long wantedRestartGeneration = 1;
-            long currentRestartGeneration = wantedRestartGeneration;
-            dockerTester.addChildNodeRepositoryNode(createNodeRepositoryNode(wantedRestartGeneration, currentRestartGeneration));
-
-            // Wait for node admin to be notified with node repo state and the docker container has been started
-            while (dockerTester.nodeAdmin.getNumberOfNodeAgents() == 0) {
-                Thread.sleep(10);
-            }
-
-            // Check that the container is started and NodeRepo has received the PATCH update
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "createContainerCommand with DockerImage { imageId=image:1.2.3 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3}");
-
-            wantedRestartGeneration = 2;
-            currentRestartGeneration = 1;
-            dockerTester.addChildNodeRepositoryNode(createNodeRepositoryNode(wantedRestartGeneration, currentRestartGeneration));
-
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "Suspend for host1.test.yahoo.com",
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", restart-vespa]");
+    public void test() {
+        try (DockerTester tester = new DockerTester()) {
+            String hostname = "host1.test.yahoo.com";
+            DockerImage dockerImage = new DockerImage("dockerImage:1.2.3");
+
+            tester.addChildNodeRepositoryNode(new NodeSpec.Builder()
+                    .hostname(hostname)
+                    .state(Node.State.active)
+                    .wantedDockerImage(dockerImage)
+                    .nodeType(NodeType.tenant)
+                    .flavor("docker")
+                    .wantedRestartGeneration(1)
+                    .currentRestartGeneration(1)
+                    .build());
+
+            tester.inOrder(tester.docker).createContainerCommand(
+                    eq(dockerImage), any(), eq(new ContainerName("host1")), eq(hostname));
+            tester.inOrder(tester.nodeRepository).updateNodeAttributes(
+                    eq(hostname), eq(new NodeAttributes().withDockerImage(dockerImage)));
+
+            // Increment wantedRestartGeneration to 2 in node-repo
+            tester.addChildNodeRepositoryNode(new NodeSpec.Builder(tester.nodeRepository.getNode(hostname))
+                    .wantedRestartGeneration(2).build());
+
+            tester.inOrder(tester.orchestrator).suspend(eq(hostname));
+            tester.inOrder(tester.docker).executeInContainerAsUser(
+                    eq(new ContainerName("host1")), any(), any(), eq(NODE_PROGRAM), eq("restart-vespa"));
+            tester.inOrder(tester.nodeRepository).updateNodeAttributes(
+                    eq(hostname), eq(new NodeAttributes().withRestartGeneration(Optional.of(2L))));
         }
     }
-
-    private NodeSpec createNodeRepositoryNode(long wantedRestartGeneration, long currentRestartGeneration) {
-        return new NodeSpec.Builder()
-                .hostname("host1.test.yahoo.com")
-                .state(Node.State.active)
-                .wantedDockerImage(new DockerImage("image:1.2.3"))
-                .wantedVespaVersion("1.2.3")
-                .nodeType(NodeType.tenant)
-                .flavor("docker")
-                .wantedRestartGeneration(wantedRestartGeneration)
-                .currentRestartGeneration(currentRestartGeneration)
-                .minCpuCores(1)
-                .minMainMemoryAvailableGb(1)
-                .minDiskAvailableGb(1)
-                .build();
-    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
deleted file mode 100644
index 298c185266b..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
-
-import com.yahoo.vespa.hosted.dockerapi.Container;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
-import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
-
-import java.util.Optional;
-
-/**
- * @author freva
- */
-public class StorageMaintainerMock extends StorageMaintainer {
-    private final CallOrderVerifier callOrderVerifier;
-
-    public StorageMaintainerMock(DockerOperations dockerOperations, CallOrderVerifier callOrderVerifier) {
-        super(dockerOperations, null, null);
-        this.callOrderVerifier = callOrderVerifier;
-    }
-
-    @Override
-    public Optional<Long> getDiskUsageFor(NodeAgentContext context) {
-        return Optional.empty();
-    }
-
-    @Override
-    public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node, Optional<Container> container) {
-    }
-
-    @Override
-    public void removeOldFilesFromNode(NodeAgentContext context) {
-    }
-
-    @Override
-    public void archiveNodeStorage(NodeAgentContext context) {
-        callOrderVerifier.add("DeleteContainerStorage with " + context.containerName());
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java
index a270585bcaa..07d3fab9534 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java
@@ -20,9 +20,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.anyVararg;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
@@ -59,8 +58,8 @@ public class AclMaintainerTest {
 
         aclMaintainer.converge();
 
-        verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), anyVararg()); //we don;t have a ip4 address for the container so no redirect either
-        verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), anyVararg());
+        verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), any()); //we don;t have a ip4 address for the container so no redirect either
+        verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), any());
     }
 
     @Test
@@ -77,8 +76,8 @@ public class AclMaintainerTest {
 
         aclMaintainer.converge();
 
-        verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), anyVararg()); //we don;t have a ip4 address for the container so no redirect either
-        verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), anyVararg());
+        verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), any()); //we don;t have a ip4 address for the container so no redirect either
+        verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), any());
     }
 
     @Test
@@ -90,7 +89,7 @@ public class AclMaintainerTest {
 
         aclMaintainer.converge();
 
-        verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(container.name), anyVararg());
+        verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(container.name), any());
     }
 
     @Test
@@ -123,8 +122,8 @@ public class AclMaintainerTest {
 
         aclMaintainer.converge();
 
-        verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), anyVararg());
-        verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), anyVararg());
+        verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), any());
+        verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), any());
     }
 
     @Test
@@ -166,8 +165,8 @@ public class AclMaintainerTest {
 
         aclMaintainer.converge();
 
-        verify(dockerOperations, never()).executeCommandInNetworkNamespace(any(), eq("ip6tables-restore"), anyVararg());
-        verify(dockerOperations, never()).executeCommandInNetworkNamespace(any(), eq("iptables-restore"), anyVararg());
+        verify(dockerOperations, never()).executeCommandInNetworkNamespace(any(), eq("ip6tables-restore"), any());
+        verify(dockerOperations, never()).executeCommandInNetworkNamespace(any(), eq("iptables-restore"), any());
     }
 
 
@@ -189,11 +188,11 @@ public class AclMaintainerTest {
 
         when(dockerOperations.executeCommandInNetworkNamespace(
                 eq(container.name),
-                eq("ip6tables-restore"), anyVararg())).thenThrow(new RuntimeException("iptables restore failed"));
+                eq("ip6tables-restore"), any())).thenThrow(new RuntimeException("iptables restore failed"));
 
         when(dockerOperations.executeCommandInNetworkNamespace(
                 eq(container.name),
-                eq("iptables-restore"), anyVararg())).thenThrow(new RuntimeException("iptables restore failed"));
+                eq("iptables-restore"), any())).thenThrow(new RuntimeException("iptables restore failed"));
 
         aclMaintainer.converge();
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditorTest.java
index 5496adf3e69..1b39014a9c6 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditorTest.java
@@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion;
 import org.junit.Assert;
 import org.junit.Test;
 
-import static org.mockito.Matchers.eq;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
index ca8435c9edd..af9e7b39657 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
@@ -33,8 +33,8 @@ import java.util.stream.Collectors;
 import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
index bb229a10f59..3860e2e9780 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
@@ -21,9 +21,9 @@ import java.util.function.Function;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.eq;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java
index 82f2408b252..6afeca6eeb5 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java
@@ -21,9 +21,9 @@ import java.util.stream.IntStream;
 import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl.TRANSITION_EXCEPTION_MESSAGE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.eq;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 00326f90caa..0e72ef15f6a 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -38,9 +38,8 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyVararg;
-import static org.mockito.Matchers.eq;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
@@ -88,17 +87,12 @@ public class NodeAgentImplTest {
 
     @Test
     public void upToDateContainerIsUntouched() {
-        final long restartGeneration = 1;
-        final long rebootGeneration = 0;
         final NodeSpec node = nodeBuilder
                 .wantedDockerImage(dockerImage)
                 .currentDockerImage(dockerImage)
                 .state(Node.State.active)
                 .wantedVespaVersion(vespaVersion)
                 .vespaVersion(vespaVersion)
-                .wantedRestartGeneration(restartGeneration)
-                .currentRestartGeneration(restartGeneration)
-                .wantedRebootGeneration(rebootGeneration)
                 .build();
 
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
@@ -121,17 +115,12 @@ public class NodeAgentImplTest {
 
     @Test
     public void verifyRemoveOldFilesIfDiskFull() {
-        final long restartGeneration = 1;
-        final long rebootGeneration = 0;
         final NodeSpec node = nodeBuilder
                 .wantedDockerImage(dockerImage)
                 .currentDockerImage(dockerImage)
                 .state(Node.State.active)
                 .wantedVespaVersion(vespaVersion)
                 .vespaVersion(vespaVersion)
-                .wantedRestartGeneration(restartGeneration)
-                .currentRestartGeneration(restartGeneration)
-                .wantedRebootGeneration(rebootGeneration)
                 .build();
 
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
@@ -182,14 +171,12 @@ public class NodeAgentImplTest {
     @Test
     public void absentContainerCausesStart() {
         final Optional<Long> restartGeneration = Optional.of(1L);
-        final long rebootGeneration = 0;
         final NodeSpec node = nodeBuilder
                 .wantedDockerImage(dockerImage)
                 .state(Node.State.active)
                 .wantedVespaVersion(vespaVersion)
                 .wantedRestartGeneration(restartGeneration.get())
                 .currentRestartGeneration(restartGeneration.get())
-                .wantedRebootGeneration(rebootGeneration)
                 .build();
 
         NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
@@ -211,26 +198,19 @@ public class NodeAgentImplTest {
         inOrder.verify(aclMaintainer, times(1)).converge();
         inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context));
         inOrder.verify(nodeRepository).updateNodeAttributes(
-                hostName, new NodeAttributes()
-                        .withRestartGeneration(restartGeneration)
-                        .withRebootGeneration(rebootGeneration)
-                        .withDockerImage(dockerImage));
+                hostName, new NodeAttributes().withDockerImage(dockerImage));
         inOrder.verify(orchestrator).resume(hostName);
     }
 
     @Test
     public void containerIsNotStoppedIfNewImageMustBePulled() {
         final DockerImage newDockerImage = new DockerImage("new-image");
-        final long wantedRestartGeneration = 2;
-        final long currentRestartGeneration = 1;
         final NodeSpec node = nodeBuilder
                 .wantedDockerImage(newDockerImage)
                 .currentDockerImage(dockerImage)
                 .state(Node.State.active)
                 .wantedVespaVersion(vespaVersion)
                 .vespaVersion(vespaVersion)
-                .wantedRestartGeneration(wantedRestartGeneration)
-                .currentRestartGeneration(currentRestartGeneration)
                 .build();
 
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
@@ -251,16 +231,12 @@ public class NodeAgentImplTest {
 
     @Test
     public void containerIsRestartedIfFlavorChanged() {
-        final long wantedRestartGeneration = 1;
-        final long currentRestartGeneration = 1;
         NodeSpec.Builder specBuilder = nodeBuilder
                 .wantedDockerImage(dockerImage)
                 .currentDockerImage(dockerImage)
                 .state(Node.State.active)
                 .wantedVespaVersion(vespaVersion)
-                .vespaVersion(vespaVersion)
-                .wantedRestartGeneration(wantedRestartGeneration)
-                .currentRestartGeneration(currentRestartGeneration);
+                .vespaVersion(vespaVersion);
 
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
         NodeSpec firstSpec = specBuilder.build();
@@ -316,16 +292,44 @@ public class NodeAgentImplTest {
     }
 
     @Test
+    public void recreatesContainerIfRebootWanted() {
+        final long wantedRebootGeneration = 2;
+        final long currentRebootGeneration = 1;
+        final NodeSpec node = nodeBuilder
+                .wantedDockerImage(dockerImage)
+                .currentDockerImage(dockerImage)
+                .state(Node.State.active)
+                .wantedVespaVersion(vespaVersion)
+                .vespaVersion(vespaVersion)
+                .wantedRebootGeneration(wantedRebootGeneration)
+                .currentRebootGeneration(currentRebootGeneration)
+                .build();
+
+        NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+        when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+        when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false);
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L));
+
+        nodeAgent.converge();
+
+        verify(orchestrator, times(1)).suspend(eq(hostName));
+        verify(dockerOperations, times(1)).removeContainer(eq(context), any());
+        verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any());
+        verify(dockerOperations, times(1)).startContainer(eq(context));
+        verify(orchestrator, times(1)).resume(eq(hostName));
+        verify(nodeRepository, times(1)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes()
+                .withRebootGeneration(wantedRebootGeneration)));
+    }
+
+    @Test
     public void failedNodeRunningContainerShouldStillBeRunning() {
-        final long restartGeneration = 1;
         final NodeSpec node = nodeBuilder
                 .wantedDockerImage(dockerImage)
                 .currentDockerImage(dockerImage)
                 .state(Node.State.failed)
                 .wantedVespaVersion(vespaVersion)
                 .vespaVersion(vespaVersion)
-                .wantedRestartGeneration(restartGeneration)
-                .currentRestartGeneration(restartGeneration)
                 .build();
 
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
@@ -341,13 +345,8 @@ public class NodeAgentImplTest {
 
     @Test
     public void readyNodeLeadsToNoAction() {
-        final long restartGeneration = 1;
-        final long rebootGeneration = 0;
         final NodeSpec node = nodeBuilder
                 .state(Node.State.ready)
-                .wantedRestartGeneration(restartGeneration)
-                .currentRestartGeneration(restartGeneration)
-                .wantedRebootGeneration(rebootGeneration)
                 .build();
 
         NodeAgentImpl nodeAgent = makeNodeAgent(null,false);
@@ -369,18 +368,12 @@ public class NodeAgentImplTest {
 
     @Test
     public void inactiveNodeRunningContainerShouldStillBeRunning() {
-        final long restartGeneration = 1;
-        final long rebootGeneration = 0;
-
         final NodeSpec node = nodeBuilder
                 .wantedDockerImage(dockerImage)
                 .currentDockerImage(dockerImage)
                 .state(Node.State.inactive)
                 .wantedVespaVersion(vespaVersion)
                 .vespaVersion(vespaVersion)
-                .wantedRestartGeneration(restartGeneration)
-                .currentRestartGeneration(restartGeneration)
-                .wantedRebootGeneration(rebootGeneration)
                 .build();
 
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
@@ -398,16 +391,10 @@ public class NodeAgentImplTest {
 
     @Test
     public void reservedNodeDoesNotUpdateNodeRepoWithVersion() {
-        final long restartGeneration = 1;
-        final long rebootGeneration = 0;
-
         final NodeSpec node = nodeBuilder
                 .wantedDockerImage(dockerImage)
                 .state(Node.State.reserved)
                 .wantedVespaVersion(vespaVersion)
-                .wantedRestartGeneration(restartGeneration)
-                .currentRestartGeneration(restartGeneration)
-                .wantedRebootGeneration(rebootGeneration)
                 .build();
 
         NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
@@ -451,10 +438,7 @@ public class NodeAgentImplTest {
         verify(orchestrator, never()).suspend(any(String.class));
         // current Docker image and vespa version should be cleared
         verify(nodeRepository, times(1)).updateNodeAttributes(
-                any(String.class), eq(new NodeAttributes()
-                        .withRestartGeneration(wantedRestartGeneration)
-                        .withRebootGeneration(0L)
-                        .withDockerImage(new DockerImage(""))));
+                eq(hostName), eq(new NodeAttributes().withDockerImage(new DockerImage(""))));
     }
 
     @Test
@@ -504,14 +488,11 @@ public class NodeAgentImplTest {
 
     @Test
     public void resumeProgramRunsUntilSuccess() {
-        final long restartGeneration = 1;
         final NodeSpec node = nodeBuilder
                 .wantedDockerImage(dockerImage)
                 .currentDockerImage(dockerImage)
                 .state(Node.State.active)
                 .vespaVersion(vespaVersion)
-                .wantedRestartGeneration(restartGeneration)
-                .currentRestartGeneration(restartGeneration)
                 .build();
 
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
@@ -570,15 +551,10 @@ public class NodeAgentImplTest {
 
     @Test
     public void start_container_subtask_failure_leads_to_container_restart() {
-        final long restartGeneration = 1;
-        final long rebootGeneration = 0;
         final NodeSpec node = nodeBuilder
                 .wantedDockerImage(dockerImage)
                 .state(Node.State.active)
                 .wantedVespaVersion(vespaVersion)
-                .wantedRestartGeneration(restartGeneration)
-                .currentRestartGeneration(restartGeneration)
-                .wantedRebootGeneration(rebootGeneration)
                 .build();
 
         NodeAgentImpl nodeAgent = spy(makeNodeAgent(null, false));
@@ -669,7 +645,7 @@ public class NodeAgentImplTest {
             assertEquals(5L, calledTimeout);
             assertArrayEquals(expectedCommand, calledCommand);
             return null;
-        }).when(dockerOperations).executeCommandInContainerAsRoot(any(), any(), anyVararg());
+        }).when(dockerOperations).executeCommandInContainerAsRoot(any(), any(), any());
 
         nodeAgent.updateContainerNodeMetrics();
     }
@@ -695,7 +671,6 @@ public class NodeAgentImplTest {
 
     @Test
     public void testRunningConfigServer() {
-        final long rebootGeneration = 0;
         final NodeSpec node = nodeBuilder
                 .nodeType(NodeType.config)
                 .wantedDockerImage(dockerImage)
@@ -721,9 +696,7 @@ public class NodeAgentImplTest {
         inOrder.verify(aclMaintainer, times(1)).converge();
         inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context));
         inOrder.verify(nodeRepository).updateNodeAttributes(
-                hostName, new NodeAttributes()
-                        .withRebootGeneration(rebootGeneration)
-                        .withDockerImage(dockerImage));
+                hostName, new NodeAttributes().withDockerImage(dockerImage));
         inOrder.verify(orchestrator).resume(hostName);
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java
index 13d06e326c7..c325a858d7c 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java
@@ -13,7 +13,7 @@ import java.util.Collections;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java
index 1e4c7482aaf..2bc64a3fdb3 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java
@@ -13,8 +13,8 @@ import java.time.Instant;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java
index 1a88af8ad0f..6151d16b69e 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java
@@ -16,8 +16,8 @@ import java.time.Instant;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyLong;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;