Rewrite integration tests to use mockito

11 files changed, 151 insertions, 485 deletions

diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/CallOrderVerifier.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/CallOrderVerifier.java
deleted file mode 100644
index df351d4cc2e..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/CallOrderVerifier.java
+++ /dev/null
@@ -1,114 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Takes in strings representing function calls with their parameters and allows to check whether a subset of calls
- * occurred in a specific order. For example, by calling {@link CallOrderVerifier#add(String)}
- * with A, B, C, D, E, D, A, F, D and G, then {@link CallOrderVerifier#verifyInOrder(String...)} with
- * A, B, D => true,
- * A, D, A => true,
- * C, D, F => true,
- * B, D, A => true
- * B, F, D, A => false,
- * C, B => false
- *
- * @author freva
- */
-public class CallOrderVerifier {
-    private static final int waitForCallOrderTimeout = 600; //ms
-
-    private final LinkedList<String> callOrder = new LinkedList<>();
-    private final Object monitor = new Object();
-
-    public void add(String call) {
-        synchronized (monitor) {
-            callOrder.add(call);
-        }
-    }
-
-    public void assertInOrder(String... functionCalls) {
-        assertInOrder(waitForCallOrderTimeout, functionCalls);
-    }
-
-    public void assertInOrder(long timeout, String... functionCalls) {
-        assertInOrderWithAssertMessage(timeout, "", functionCalls);
-    }
-
-    public void assertInOrderWithAssertMessage(String assertMessage, String... functionCalls) {
-        assertInOrderWithAssertMessage(waitForCallOrderTimeout, assertMessage, functionCalls);
-    }
-
-    public void assertInOrderWithAssertMessage(long timeout, String assertMessage, String... functionCalls) {
-        boolean inOrder = verifyInOrder(timeout, functionCalls);
-        if ( ! inOrder && ! assertMessage.isEmpty())
-            System.err.println(assertMessage);
-        assertTrue(toString(), inOrder);
-    }
-
-    /**
-     * Checks if list of function calls occur in order given within a timeout
-     * @param timeout Max number of milliseconds to check for if function calls occur in order
-     * @param functionCalls The expected order of function calls
-     * @return true if the actual order of calls was equal to the order provided within timeout, false otherwise.
-     */
-    private boolean verifyInOrder(long timeout, String... functionCalls) {
-        final long startTime = System.currentTimeMillis();
-        while (System.currentTimeMillis() - startTime < timeout) {
-            if (verifyInOrder(functionCalls)) {
-                return true;
-            }
-            try {
-                Thread.sleep(10);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        }
-
-        return false;
-    }
-
-    private boolean verifyInOrder(String... functionCalls) {
-        int pos = 0;
-        synchronized (monitor) {
-            for (String functionCall : functionCalls) {
-                int temp = indexOf(callOrder.listIterator(pos), functionCall);
-                if (temp == -1) {
-                    System.out.println("Function call '" + functionCall +
-                            "' never made at position " + pos);
-                    return false;
-                }
-                pos += temp;
-            }
-        }
-
-        return true;
-    }
-
-    /**
-     * Finds the first index of needle in haystack after a given position.
-     * @param iter Iterator to search in
-     * @param search Element to find in iterator
-     * @return Index of the next search in  after startPos, -1 if not found
-     */
-    private int indexOf(Iterator<String> iter, String search) {
-        for (int i = 0; iter.hasNext(); i++) {
-            if (search.equals(iter.next())) {
-                return i;
-            }
-        }
-
-        return -1;
-    }
-
-    @Override
-    public String toString() {
-        synchronized (monitor) {
-            return callOrder.toString();
-        }
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
index d7e0fe3d8e5..b3bf2282988 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
@@ -3,22 +3,32 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.provision.Node;
 import org.junit.Test;
 
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
 /**
  * @author freva
  */
 public class DockerFailTest {
 
     @Test
-    public void dockerFailTest() throws Exception {
-        try (DockerTester dockerTester = new DockerTester()) {
-            NodeSpec nodeSpec = new NodeSpec.Builder()
-                    .hostname("host1.test.yahoo.com")
-                    .wantedDockerImage(new DockerImage("dockerImage"))
+    public void dockerFailTest() {
+        try (DockerTester tester = new DockerTester()) {
+            final DockerImage dockerImage = new DockerImage("dockerImage");
+            final ContainerName containerName = new ContainerName("host1");
+            final String hostname = "host1.test.yahoo.com";
+            tester.addChildNodeRepositoryNode(new NodeSpec.Builder()
+                    .hostname(hostname)
+                    .wantedDockerImage(dockerImage)
+                    .currentDockerImage(dockerImage)
                     .state(Node.State.active)
                     .nodeType(NodeType.tenant)
                     .flavor("docker")
@@ -27,24 +37,22 @@ public class DockerFailTest {
                     .minCpuCores(1)
                     .minMainMemoryAvailableGb(1)
                     .minDiskAvailableGb(1)
-                    .build();
-            dockerTester.addChildNodeRepositoryNode(nodeSpec);
+                    .build());
 
-            // Wait for node admin to be notified with node repo state and the docker container has been started
-            while (dockerTester.nodeAdmin.getNumberOfNodeAgents() == 0) {
-                Thread.sleep(10);
-            }
+            tester.inOrder(tester.docker).createContainerCommand(
+                    eq(dockerImage), eq(ContainerResources.from(1, 1)), eq(containerName), eq(hostname));
+            tester.inOrder(tester.docker).executeInContainerAsUser(
+                    eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume"));
 
-            dockerTester.callOrderVerifier.assertInOrder(1200,
-                    "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+            tester.docker.deleteContainer(new ContainerName("host1"));
 
-            dockerTester.dockerMock.deleteContainer(new ContainerName("host1"));
+            tester.inOrder(tester.docker).deleteContainer(eq(containerName));
+            tester.inOrder(tester.docker).createContainerCommand(
+                    eq(dockerImage), eq(ContainerResources.from(1, 1)), eq(containerName), eq(hostname));
+            tester.inOrder(tester.docker).executeInContainerAsUser(
+                    eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume"));
 
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "deleteContainer with ContainerName { name=host1 }",
-                    "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+            verify(tester.nodeRepository, never()).updateNodeAttributes(any(), any());
         }
     }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
index 26843b119de..75bdaed60cf 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
@@ -13,7 +13,6 @@ import java.net.InetAddress;
 import java.nio.file.Path;
 import java.time.Duration;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -27,13 +26,8 @@ import java.util.OptionalLong;
  */
 public class DockerMock implements Docker {
     private final Map<ContainerName, Container> containersByContainerName = new HashMap<>();
-    public final CallOrderVerifier callOrderVerifier;
     private static final Object monitor = new Object();
 
-    public DockerMock(CallOrderVerifier callOrderVerifier) {
-        this.callOrderVerifier = callOrderVerifier;
-    }
-
     @Override
     public CreateContainerCommand createContainerCommand(
             DockerImage dockerImage,
@@ -41,8 +35,6 @@ public class DockerMock implements Docker {
             ContainerName containerName,
             String hostName) {
         synchronized (monitor) {
-            callOrderVerifier.add("createContainerCommand with " + dockerImage +
-                    ", HostName: " + hostName + ", " + containerName);
             containersByContainerName.put(
                     containerName, new Container(hostName, dockerImage, containerResources, containerName, Container.State.RUNNING, 2));
         }
@@ -64,15 +56,12 @@ public class DockerMock implements Docker {
 
     @Override
     public void startContainer(ContainerName containerName) {
-        synchronized (monitor) {
-            callOrderVerifier.add("startContainer with " + containerName);
-        }
+
     }
 
     @Override
     public void stopContainer(ContainerName containerName) {
         synchronized (monitor) {
-            callOrderVerifier.add("stopContainer with " + containerName);
             Container container = containersByContainerName.get(containerName);
             containersByContainerName.put(containerName,
                             new Container(container.hostname, container.image, container.resources, container.name, Container.State.EXITED, 0));
@@ -82,7 +71,6 @@ public class DockerMock implements Docker {
     @Override
     public void deleteContainer(ContainerName containerName) {
         synchronized (monitor) {
-            callOrderVerifier.add("deleteContainer with " + containerName);
             containersByContainerName.remove(containerName);
         }
     }
@@ -97,7 +85,6 @@ public class DockerMock implements Docker {
     @Override
     public boolean pullImageAsyncIfNeeded(DockerImage image) {
         synchronized (monitor) {
-            callOrderVerifier.add("pullImageAsyncIfNeeded with " + image);
             return false;
         }
     }
@@ -109,9 +96,6 @@ public class DockerMock implements Docker {
 
     @Override
     public ProcessResult executeInContainerAsUser(ContainerName containerName, String user, OptionalLong timeout, String... args) {
-        synchronized (monitor) {
-            callOrderVerifier.add("executeInContainer " + containerName.asString() + " as " + user + ", args: " + Arrays.toString(args));
-        }
         return new ProcessResult(0, null, "");
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
index aa40d5d805b..6ed51657d6e 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
@@ -9,8 +9,10 @@ import com.yahoo.system.ProcessExecuter;
 import com.yahoo.vespa.hosted.dockerapi.Docker;
 import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl;
+import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent;
@@ -19,6 +21,8 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl;
 import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.test.file.TestFileSystem;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
 
 import java.nio.file.FileSystem;
 import java.nio.file.Path;
@@ -30,8 +34,10 @@ import java.util.Optional;
 import java.util.function.Function;
 
 import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck;
-import static org.mockito.Matchers.any;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
 import static org.mockito.Mockito.when;
 
 /**
@@ -45,16 +51,19 @@ public class DockerTester implements AutoCloseable {
     static final String NODE_PROGRAM = PATH_TO_VESPA_HOME.resolve("bin/vespa-nodectl").toString();
     static final HostName HOST_HOSTNAME = HostName.from("host.test.yahoo.com");
 
-    final CallOrderVerifier callOrderVerifier = new CallOrderVerifier();
-    final Docker dockerMock = new DockerMock(callOrderVerifier);
-    final NodeRepoMock nodeRepositoryMock = new NodeRepoMock(callOrderVerifier);
+    final Docker docker = spy(new DockerMock());
+    final NodeRepoMock nodeRepository = spy(new NodeRepoMock());
+    final Orchestrator orchestrator = mock(Orchestrator.class);
+    final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class);
+    final InOrder inOrder = Mockito.inOrder(docker, nodeRepository, orchestrator, storageMaintainer);
+
     final NodeAdminStateUpdaterImpl nodeAdminStateUpdater;
     final NodeAdminImpl nodeAdmin;
-    private final OrchestratorMock orchestratorMock = new OrchestratorMock(callOrderVerifier);
-    private final FileSystem fileSystem = TestFileSystem.create();
 
 
     DockerTester() {
+        when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.empty());
+
         IPAddressesMock ipAddresses = new IPAddressesMock();
         ipAddresses.addAddress(HOST_HOSTNAME.value(), "1.1.1.1");
         ipAddresses.addAddress(HOST_HOSTNAME.value(), "f000::");
@@ -71,25 +80,25 @@ public class DockerTester implements AutoCloseable {
                 .wantedRestartGeneration(1L)
                 .currentRestartGeneration(1L)
                 .build();
-        nodeRepositoryMock.updateNodeRepositoryNode(hostSpec);
+        nodeRepository.updateNodeRepositoryNode(hostSpec);
 
         Clock clock = Clock.systemUTC();
-        DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, processExecuter, node -> "", Collections.emptyList(), ipAddresses);
-        StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, callOrderVerifier);
+        FileSystem fileSystem = TestFileSystem.create();
+        DockerOperations dockerOperations = new DockerOperationsImpl(docker, processExecuter, node -> "", Collections.emptyList(), ipAddresses);
 
         MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
         Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(
-                new NodeAgentContextImpl.Builder(hostName).fileSystem(fileSystem).build(), nodeRepositoryMock,
-                orchestratorMock, dockerOperations, storageMaintainer, clock, NODE_AGENT_SCAN_INTERVAL, Optional.empty(), Optional.empty());
+                new NodeAgentContextImpl.Builder(hostName).fileSystem(fileSystem).build(), nodeRepository,
+                orchestrator, dockerOperations, storageMaintainer, clock, NODE_AGENT_SCAN_INTERVAL, Optional.empty(), Optional.empty());
         nodeAdmin = new NodeAdminImpl(nodeAgentFactory, Optional.empty(), mr, Clock.systemUTC());
-        nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock,
+        nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepository, orchestrator,
                 nodeAdmin, HOST_HOSTNAME, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL);
         nodeAdminStateUpdater.start();
     }
 
     /** Adds a node to node-repository mock that is running on this host */
     void addChildNodeRepositoryNode(NodeSpec nodeSpec) {
-        nodeRepositoryMock.updateNodeRepositoryNode(new NodeSpec.Builder(nodeSpec)
+        nodeRepository.updateNodeRepositoryNode(new NodeSpec.Builder(nodeSpec)
                 .parentHostname(HOST_HOSTNAME.value())
                 .build());
     }
@@ -98,4 +107,8 @@ public class DockerTester implements AutoCloseable {
     public void close() {
         nodeAdminStateUpdater.stop();
     }
+
+    public <T> T inOrder(T t) {
+        return inOrder.verify(t, timeout(1000));
+    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
index e130cd0d475..48fdc564aac 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
@@ -3,24 +3,30 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.provision.Node;
 import org.junit.Test;
 
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+
 /**
  * @author freva
  */
 public class MultiDockerTest {
 
     @Test
-    public void test() throws InterruptedException {
-        try (DockerTester dockerTester = new DockerTester()) {
-            addAndWaitForNode(dockerTester, "host1.test.yahoo.com", new DockerImage("image1"));
+    public void test() {
+        try (DockerTester tester = new DockerTester()) {
+            addAndWaitForNode(tester, "host1.test.yahoo.com", new DockerImage("image1"));
             NodeSpec nodeSpec2 = addAndWaitForNode(
-                    dockerTester, "host2.test.yahoo.com", new DockerImage("image2"));
+                    tester, "host2.test.yahoo.com", new DockerImage("image2"));
 
-            dockerTester.addChildNodeRepositoryNode(
+            tester.addChildNodeRepositoryNode(
                     new NodeSpec.Builder(nodeSpec2)
                             .state(Node.State.dirty)
                             .minCpuCores(1)
@@ -28,66 +34,37 @@ public class MultiDockerTest {
                             .minDiskAvailableGb(1)
                             .build());
 
-            // Wait until it is marked ready
-            while (dockerTester.nodeRepositoryMock.getOptionalNode(nodeSpec2.getHostname())
-                    .filter(node -> node.getState() != Node.State.ready).isPresent()) {
-                Thread.sleep(10);
-            }
-
-            addAndWaitForNode(dockerTester, "host3.test.yahoo.com", new DockerImage("image1"));
-
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "createContainerCommand with DockerImage { imageId=image1 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]",
-
-                    "createContainerCommand with DockerImage { imageId=image2 }, HostName: host2.test.yahoo.com, ContainerName { name=host2 }",
-                    "executeInContainer host2 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]",
-
-                    "stopContainer with ContainerName { name=host2 }",
-                    "deleteContainer with ContainerName { name=host2 }",
+            tester.inOrder(tester.docker).deleteContainer(eq(new ContainerName("host2")));
+            tester.inOrder(tester.storageMaintainer).archiveNodeStorage(
+                    argThat(context -> context.containerName().equals(new ContainerName("host2"))));
+            tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.getHostname()), eq(Node.State.ready));
 
-                    "createContainerCommand with DockerImage { imageId=image1 }, HostName: host3.test.yahoo.com, ContainerName { name=host3 }",
-                    "executeInContainer host3 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
-
-            dockerTester.callOrderVerifier.assertInOrderWithAssertMessage(
-                    "Maintainer did not receive call to delete application storage",
-                    "deleteContainer with ContainerName { name=host2 }",
-                    "DeleteContainerStorage with ContainerName { name=host2 }");
-
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1}",
-                    "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2}",
-                    "setNodeState host2.test.yahoo.com to ready",
-                    "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1}");
+            addAndWaitForNode(tester, "host3.test.yahoo.com", new DockerImage("image1"));
         }
     }
 
-    private NodeSpec addAndWaitForNode(DockerTester tester, String hostName, DockerImage dockerImage) throws InterruptedException {
+    private NodeSpec addAndWaitForNode(DockerTester tester, String hostName, DockerImage dockerImage) {
         NodeSpec nodeSpec = new NodeSpec.Builder()
                 .hostname(hostName)
                 .wantedDockerImage(dockerImage)
-                .wantedVespaVersion("1.2.3")
                 .state(Node.State.active)
                 .nodeType(NodeType.tenant)
                 .flavor("docker")
                 .wantedRestartGeneration(1L)
                 .currentRestartGeneration(1L)
-                .minCpuCores(1)
-                .minMainMemoryAvailableGb(1)
+                .minCpuCores(2)
+                .minMainMemoryAvailableGb(4)
                 .minDiskAvailableGb(1)
                 .build();
 
         tester.addChildNodeRepositoryNode(nodeSpec);
 
-        // Wait for node admin to be notified with node repo state and the docker container has been started
-        while (tester.nodeAdmin.getNumberOfNodeAgents() + 1 != tester.nodeRepositoryMock.getNumberOfContainerSpecs()) {
-            Thread.sleep(10);
-        }
-
         ContainerName containerName = ContainerName.fromHostname(hostName);
-        tester.callOrderVerifier.assertInOrder(
-                "createContainerCommand with " + dockerImage + ", HostName: " + hostName + ", " + containerName,
-                "executeInContainer " + containerName.asString() + " as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+        tester.inOrder(tester.docker).createContainerCommand(
+                eq(dockerImage), eq(ContainerResources.from(2, 4)), eq(containerName), eq(hostName));
+        tester.inOrder(tester.docker).executeInContainerAsUser(
+                eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume"));
+        tester.inOrder(tester.nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withDockerImage(dockerImage)));
 
         return nodeSpec;
     }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
index d25d79ab457..e8b423e73ac 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttribu
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
 import com.yahoo.vespa.hosted.provision.Node;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -23,13 +24,6 @@ public class NodeRepoMock implements NodeRepository {
     private static final Object monitor = new Object();
 
     private final Map<String, NodeSpec> nodeRepositoryNodesByHostname = new HashMap<>();
-    private final Map<String, Acl> acls = new HashMap<>();
-
-    private final CallOrderVerifier callOrderVerifier;
-
-    public NodeRepoMock(CallOrderVerifier callOrderVerifier) {
-        this.callOrderVerifier = callOrderVerifier;
-    }
 
     @Override
     public void addNodes(List<AddNode> nodes) { }
@@ -52,15 +46,15 @@ public class NodeRepoMock implements NodeRepository {
 
     @Override
     public Map<String, Acl> getAcls(String hostname) {
-        synchronized (monitor) {
-            return acls;
-        }
+        return Collections.emptyMap();
     }
 
     @Override
     public void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes) {
         synchronized (monitor) {
-            callOrderVerifier.add("updateNodeAttributes with HostName: " + hostName + ", " + nodeAttributes);
+            updateNodeRepositoryNode(new NodeSpec.Builder(getNode(hostName))
+                    .updateFromNodeAttributes(nodeAttributes)
+                    .build());
         }
     }
 
@@ -70,7 +64,6 @@ public class NodeRepoMock implements NodeRepository {
             updateNodeRepositoryNode(new NodeSpec.Builder(getNode(hostName))
                     .state(nodeState)
                     .build());
-            callOrderVerifier.add("setNodeState " + hostName + " to " + nodeState);
         }
     }
 
@@ -79,13 +72,9 @@ public class NodeRepoMock implements NodeRepository {
 
     }
 
-    public void updateNodeRepositoryNode(NodeSpec nodeSpec) {
-        nodeRepositoryNodesByHostname.put(nodeSpec.getHostname(), nodeSpec);
-    }
-
-    public int getNumberOfContainerSpecs() {
+    void updateNodeRepositoryNode(NodeSpec nodeSpec) {
         synchronized (monitor) {
-            return nodeRepositoryNodesByHostname.size();
+            nodeRepositoryNodesByHostname.put(nodeSpec.getHostname(), nodeSpec);
         }
     }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
deleted file mode 100644
index da743c40e8b..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
-
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.dockerapi.DockerImage;
-import com.yahoo.vespa.hosted.provision.Node;
-import org.junit.Test;
-
-/**
- * Test NodeState transitions in NodeRepository
- *
- * @author freva
- */
-public class NodeStateTest {
-    private final NodeSpec initialNodeSpec = new NodeSpec.Builder()
-            .hostname("host1.test.yahoo.com")
-            .wantedDockerImage(new DockerImage("dockerImage"))
-            .state(Node.State.active)
-            .nodeType(NodeType.tenant)
-            .flavor("docker")
-            .wantedRestartGeneration(1L)
-            .currentRestartGeneration(1L)
-            .minCpuCores(1)
-            .minMainMemoryAvailableGb(1)
-            .minDiskAvailableGb(1)
-            .build();
-
-    private void setup(DockerTester tester) throws InterruptedException {
-        tester.addChildNodeRepositoryNode(initialNodeSpec);
-
-        // Wait for node admin to be notified with node repo state and the docker container has been started
-        while (tester.nodeAdmin.getNumberOfNodeAgents() == 0) {
-            Thread.sleep(10);
-        }
-
-        tester.callOrderVerifier.assertInOrder(
-                "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
-    }
-
-
-    @Test
-    public void activeToDirty() throws InterruptedException {
-        try (DockerTester dockerTester = new DockerTester()) {
-            setup(dockerTester);
-            // Change node state to dirty
-            dockerTester.addChildNodeRepositoryNode(new NodeSpec.Builder(initialNodeSpec)
-                    .state(Node.State.dirty)
-                    .minCpuCores(1)
-                    .minMainMemoryAvailableGb(1)
-                    .minDiskAvailableGb(1)
-                    .build());
-
-            // Wait until it is marked ready
-            while (dockerTester.nodeRepositoryMock.getOptionalNode(initialNodeSpec.getHostname())
-                    .filter(node -> node.getState() != Node.State.ready).isPresent()) {
-                Thread.sleep(10);
-            }
-
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", stop]",
-                    "stopContainer with ContainerName { name=host1 }",
-                    "deleteContainer with ContainerName { name=host1 }");
-        }
-    }
-
-    @Test
-    public void activeToInactiveToActive() throws InterruptedException {
-
-        try (DockerTester dockerTester = new DockerTester()) {
-            setup(dockerTester);
-
-            DockerImage newDockerImage = new DockerImage("newDockerImage");
-
-            // Change node state to inactive and change the wanted docker image
-            dockerTester.addChildNodeRepositoryNode(new NodeSpec.Builder(initialNodeSpec)
-                    .wantedDockerImage(newDockerImage)
-                    .state(Node.State.inactive)
-                    .minCpuCores(1)
-                    .minMainMemoryAvailableGb(1)
-                    .minDiskAvailableGb(1)
-                    .build());
-
-            dockerTester.callOrderVerifier.assertInOrderWithAssertMessage(
-                    "Node set to inactive, but no stop/delete call received",
-                    "stopContainer with ContainerName { name=host1 }",
-                    "deleteContainer with ContainerName { name=host1 }");
-
-
-            // Change node state to active
-            dockerTester.addChildNodeRepositoryNode(new NodeSpec.Builder(initialNodeSpec)
-                    .wantedDockerImage(newDockerImage)
-                    .state(Node.State.active)
-                    .minCpuCores(1)
-                    .minMainMemoryAvailableGb(1)
-                    .minDiskAvailableGb(1)
-                    .build());
-
-            // Check that the container is started again after the delete call
-            dockerTester.callOrderVerifier.assertInOrderWithAssertMessage(
-                    "Node not started again after being put to active state",
-                    "deleteContainer with ContainerName { name=host1 }",
-                    "createContainerCommand with DockerImage { imageId=newDockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
-        }
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java
deleted file mode 100644
index 469022cec56..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
-
-import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
-
-import java.util.List;
-
-/**
- * Mock with some simple logic
- *
- * @author dybis
- */
-public class OrchestratorMock implements Orchestrator {
-    private final CallOrderVerifier callOrderVerifier;
-
-    OrchestratorMock(CallOrderVerifier callOrderVerifier) {
-        this.callOrderVerifier = callOrderVerifier;
-    }
-
-    @Override
-    public void suspend(String hostName) {
-        callOrderVerifier.add("Suspend for " + hostName);
-    }
-
-    @Override
-    public void resume(String hostName) {
-        callOrderVerifier.add("Resume for " + hostName);
-    }
-
-    @Override
-    public void suspend(String parentHostName, List<String> hostNames) {
-        callOrderVerifier.add("Suspend with parent: " + parentHostName + " and hostnames: " + hostNames);
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
index 778443295bf..75fefde427d 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
@@ -2,16 +2,24 @@
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin;
 import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater;
-import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl;
 import com.yahoo.vespa.hosted.provision.Node;
-import org.junit.Ignore;
 import org.junit.Test;
 
+import java.util.Arrays;
+import java.util.OptionalLong;
+
+import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.HOST_HOSTNAME;
+import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.NODE_PROGRAM;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
 
 /**
  * Tests rebooting of Docker host
@@ -20,47 +28,33 @@ import static org.junit.Assert.assertTrue;
  */
 public class RebootTest {
 
-    @Test
-    @Ignore
-    public void test() throws InterruptedException {
-        try (DockerTester dockerTester = new DockerTester()) {
-
-            dockerTester.addChildNodeRepositoryNode(createNodeRepositoryNode());
-
-            // Wait for node admin to be notified with node repo state and the docker container has been started
-            while (dockerTester.nodeAdmin.getNumberOfNodeAgents() == 0) {
-                Thread.sleep(10);
-            }
+    private final String hostname = "host1.test.yahoo.com";
+    private final DockerImage dockerImage = new DockerImage("dockerImage");
 
-            // Check that the container is started and NodeRepo has received the PATCH update
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=null,  dockerImage=dockerImage, vespaVersion='null'}");
-
-            NodeAdminStateUpdaterImpl updater = dockerTester.nodeAdminStateUpdater;
-//            assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED),
-//                       is(Optional.of("Not all node agents are frozen.")));
-
-            updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED);
+    @Test
+    public void test() {
+        try (DockerTester tester = new DockerTester()) {
+            tester.addChildNodeRepositoryNode(createNodeRepositoryNode());
 
-            NodeAdmin nodeAdmin = dockerTester.nodeAdmin;
-            // Wait for node admin to be frozen
-            while ( ! nodeAdmin.isFrozen()) {
-                System.out.println("Node admin not frozen yet");
-                Thread.sleep(10);
-            }
+            tester.inOrder(tester.docker).createContainerCommand(
+                    eq(dockerImage), eq(ContainerResources.from(0, 0)), eq(new ContainerName("host1")), eq(hostname));
 
-            assertTrue(nodeAdmin.setFrozen(false));
+            try {
+                tester.nodeAdminStateUpdater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED);
+            } catch (RuntimeException ignored) { }
 
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "executeInContainer with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", stop]");
+            tester.inOrder(tester.orchestrator).suspend(
+                    eq(HOST_HOSTNAME.value()), eq(Arrays.asList(hostname, HOST_HOSTNAME.value())));
+            tester.inOrder(tester.docker).executeInContainerAsUser(
+                    eq(new ContainerName("host1")), eq("root"), eq(OptionalLong.empty()), eq(NODE_PROGRAM), eq("stop"));
+            assertTrue(tester.nodeAdmin.setFrozen(true));
         }
     }
 
     private NodeSpec createNodeRepositoryNode() {
         return new NodeSpec.Builder()
-                .hostname("host1.test.yahoo.com")
-                .wantedDockerImage(new DockerImage("dockerImage"))
+                .hostname(hostname)
+                .wantedDockerImage(dockerImage)
                 .state(Node.State.active)
                 .nodeType(NodeType.tenant)
                 .flavor("docker")
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
index 2e290e014c2..2b8727529f2 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
@@ -2,11 +2,19 @@
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.provision.Node;
 import org.junit.Test;
 
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.NODE_PROGRAM;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+
 /**
  * Tests that different wanted and current restart generation leads to execution of restart command
  *
@@ -15,46 +23,35 @@ import org.junit.Test;
 public class RestartTest {
 
     @Test
-    public void test() throws InterruptedException {
-        try (DockerTester dockerTester = new DockerTester()) {
-
-            long wantedRestartGeneration = 1;
-            long currentRestartGeneration = wantedRestartGeneration;
-            dockerTester.addChildNodeRepositoryNode(createNodeRepositoryNode(wantedRestartGeneration, currentRestartGeneration));
-
-            // Wait for node admin to be notified with node repo state and the docker container has been started
-            while (dockerTester.nodeAdmin.getNumberOfNodeAgents() == 0) {
-                Thread.sleep(10);
-            }
-
-            // Check that the container is started and NodeRepo has received the PATCH update
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "createContainerCommand with DockerImage { imageId=image:1.2.3 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                    "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3}");
-
-            wantedRestartGeneration = 2;
-            currentRestartGeneration = 1;
-            dockerTester.addChildNodeRepositoryNode(createNodeRepositoryNode(wantedRestartGeneration, currentRestartGeneration));
-
-            dockerTester.callOrderVerifier.assertInOrder(
-                    "Suspend for host1.test.yahoo.com",
-                    "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", restart-vespa]");
+    public void test() {
+        try (DockerTester tester = new DockerTester()) {
+            String hostname = "host1.test.yahoo.com";
+            DockerImage dockerImage = new DockerImage("dockerImage:1.2.3");
+
+            tester.addChildNodeRepositoryNode(new NodeSpec.Builder()
+                    .hostname(hostname)
+                    .state(Node.State.active)
+                    .wantedDockerImage(dockerImage)
+                    .nodeType(NodeType.tenant)
+                    .flavor("docker")
+                    .wantedRestartGeneration(1)
+                    .currentRestartGeneration(1)
+                    .build());
+
+            tester.inOrder(tester.docker).createContainerCommand(
+                    eq(dockerImage), any(), eq(new ContainerName("host1")), eq(hostname));
+            tester.inOrder(tester.nodeRepository).updateNodeAttributes(
+                    eq(hostname), eq(new NodeAttributes().withDockerImage(dockerImage)));
+
+            // Increment wantedRestartGeneration to 2 in node-repo
+            tester.addChildNodeRepositoryNode(new NodeSpec.Builder(tester.nodeRepository.getNode(hostname))
+                    .wantedRestartGeneration(2).build());
+
+            tester.inOrder(tester.orchestrator).suspend(eq(hostname));
+            tester.inOrder(tester.docker).executeInContainerAsUser(
+                    eq(new ContainerName("host1")), any(), any(), eq(NODE_PROGRAM), eq("restart-vespa"));
+            tester.inOrder(tester.nodeRepository).updateNodeAttributes(
+                    eq(hostname), eq(new NodeAttributes().withRestartGeneration(Optional.of(2L))));
         }
     }
-
-    private NodeSpec createNodeRepositoryNode(long wantedRestartGeneration, long currentRestartGeneration) {
-        return new NodeSpec.Builder()
-                .hostname("host1.test.yahoo.com")
-                .state(Node.State.active)
-                .wantedDockerImage(new DockerImage("image:1.2.3"))
-                .wantedVespaVersion("1.2.3")
-                .nodeType(NodeType.tenant)
-                .flavor("docker")
-                .wantedRestartGeneration(wantedRestartGeneration)
-                .currentRestartGeneration(currentRestartGeneration)
-                .minCpuCores(1)
-                .minMainMemoryAvailableGb(1)
-                .minDiskAvailableGb(1)
-                .build();
-    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
deleted file mode 100644
index 298c185266b..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
-
-import com.yahoo.vespa.hosted.dockerapi.Container;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
-import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
-
-import java.util.Optional;
-
-/**
- * @author freva
- */
-public class StorageMaintainerMock extends StorageMaintainer {
-    private final CallOrderVerifier callOrderVerifier;
-
-    public StorageMaintainerMock(DockerOperations dockerOperations, CallOrderVerifier callOrderVerifier) {
-        super(dockerOperations, null, null);
-        this.callOrderVerifier = callOrderVerifier;
-    }
-
-    @Override
-    public Optional<Long> getDiskUsageFor(NodeAgentContext context) {
-        return Optional.empty();
-    }
-
-    @Override
-    public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node, Optional<Container> container) {
-    }
-
-    @Override
-    public void removeOldFilesFromNode(NodeAgentContext context) {
-    }
-
-    @Override
-    public void archiveNodeStorage(NodeAgentContext context) {
-        callOrderVerifier.add("DeleteContainerStorage with " + context.containerName());
-    }
-}