Remove ContainerOperationsImpl

11 files changed, 254 insertions, 828 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerOperationsImpl.java
deleted file mode 100644
index 4f57a03786b..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerOperationsImpl.java
+++ /dev/null
@@ -1,387 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.docker;
-
-import com.google.common.net.InetAddresses;
-import com.yahoo.config.provision.DockerImage;
-import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.config.provision.SystemName;
-import com.yahoo.vespa.hosted.dockerapi.Container;
-import com.yahoo.vespa.hosted.dockerapi.ContainerEngine;
-import com.yahoo.vespa.hosted.dockerapi.ContainerId;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
-import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
-import com.yahoo.vespa.hosted.dockerapi.ContainerStats;
-import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
-import com.yahoo.vespa.hosted.dockerapi.RegistryCredentials;
-import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
-import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
-import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
-import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses;
-import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion;
-import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
-import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
-
-import java.net.InetAddress;
-import java.nio.file.FileSystem;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-import java.util.OptionalLong;
-import java.util.Random;
-import java.util.Set;
-import java.util.logging.Logger;
-import java.util.stream.Stream;
-
-/**
- * Class that wraps the ContainerEngine class and have some tools related to running programs in containerEngine.
- *
- * @author Haakon Dybdahl
- */
-// TODO: Remove when Podman becomes the only implementation in use
-public class ContainerOperationsImpl implements ContainerOperations {
-
-    private static final Logger logger = Logger.getLogger(ContainerOperationsImpl.class.getName());
-
-    static final String MANAGER_NAME = "node-admin";
-
-    private static final InetAddress IPV6_NPT_PREFIX = InetAddresses.forString("fd00::");
-    private static final InetAddress IPV4_NPT_PREFIX = InetAddresses.forString("172.17.0.0");
-    private static final String ETC_MACHINE_ID = "/etc/machine-id";
-
-    private static final Random random = new Random(System.nanoTime());
-
-    private final ContainerEngine containerEngine;
-    private final Terminal terminal;
-    private final IPAddresses ipAddresses;
-    private final FileSystem fileSystem;
-
-    public ContainerOperationsImpl(ContainerEngine containerEngine, Terminal terminal, IPAddresses ipAddresses, FileSystem fileSystem) {
-        this.containerEngine = containerEngine;
-        this.terminal = terminal;
-        this.ipAddresses = ipAddresses;
-        this.fileSystem = fileSystem;
-    }
-
-    @Override
-    public void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources) {
-        context.log(logger, "Creating container");
-
-        ContainerEngine.CreateContainerCommand command = containerEngine.createContainerCommand(
-                context.node().wantedDockerImage().get(), context.containerName())
-                .withHostName(context.node().hostname())
-                .withResources(containerResources)
-                .withManagedBy(MANAGER_NAME)
-                // The inet6 option is needed to prefer AAAA records with gethostbyname(3), used by (at least) a yca package
-                // TODO: Try to remove this
-                .withDnsOption("inet6")
-                .withUlimit("nofile", 262_144, 262_144)
-                // The nproc aka RLIMIT_NPROC resource limit works as follows:
-                //  - A process has a (soft) nproc limit, either inherited by the parent or changed with setrlimit(2).
-                //    In bash, a command's limit can be viewed and set with ulimit(1).
-                //  - When a process forks, the number of processes on the host (across all containers) with
-                //    the same real user ID is compared with the limit, and if above the limit, return EAGAIN.
-                //
-                // From experience our Vespa processes require a high limit, say 400k. For all other processes,
-                // we would like to use a much lower limit, say 32k.
-                .withUlimit("nproc", 409_600, 409_600)
-                .withUlimit("core", -1, -1)
-                .withAddCapability("SYS_PTRACE") // Needed for gcore, pstack etc.
-                .withAddCapability("SYS_ADMIN")  // Needed for perf
-                .withAddCapability("SYS_NICE");  // Needed for set_mempolicy to work
-
-        // Proxy and controller require new privileges to bind port 443
-        if (context.nodeType() != NodeType.proxy && context.nodeType() != NodeType.controller)
-            command.withSecurityOpt("no-new-privileges");
-
-        if (context.node().membership().map(m -> m.type().hasContent()).orElse(false))
-            command.withSecurityOpt("seccomp=unconfined");
-
-        ContainerNetworkMode networkMode = context.networkMode();
-        command.withNetworkMode(networkMode.networkName());
-
-        if (networkMode == ContainerNetworkMode.NPT) {
-            Optional<? extends InetAddress> ipV4Local = ipAddresses.getIPv4Address(context.node().hostname());
-            Optional<? extends InetAddress> ipV6Local = ipAddresses.getIPv6Address(context.node().hostname());
-
-            assertEqualIpAddresses(context.hostname(), ipV4Local, context.node().ipAddresses(), IPVersion.IPv4);
-            assertEqualIpAddresses(context.hostname(), ipV6Local, context.node().ipAddresses(), IPVersion.IPv6);
-
-            if (ipV4Local.isEmpty() && ipV6Local.isEmpty()) {
-                throw new ConvergenceException("Container " + context.node().hostname() + " with " + networkMode +
-                        " networking must have at least 1 IP address, but found none");
-            }
-
-            ipV6Local = ipV6Local.map(ip -> IPAddresses.prefixTranslate(ip, IPV6_NPT_PREFIX, 8));
-            ipV6Local.ifPresent(command::withIpAddress);
-
-            ipV4Local = ipV4Local.map(ip -> IPAddresses.prefixTranslate(ip, IPV4_NPT_PREFIX, 2));
-            ipV4Local.ifPresent(command::withIpAddress);
-
-            addEtcHosts(containerData, context.node().hostname(), ipV4Local, ipV6Local);
-        } else if (networkMode == ContainerNetworkMode.LOCAL) {
-            var ipv4Address = ipAddresses.getIPv4Address(context.node().hostname())
-                                         .orElseThrow(() -> new IllegalArgumentException("No IPv4 address could be resolved from '" + context.hostname()+ "'"));
-            command.withIpAddress(ipv4Address);
-        }
-
-        UnixPath machineIdPath = new UnixPath(context.pathOnHostFromPathInNode(ETC_MACHINE_ID));
-        if (!machineIdPath.exists()) {
-            String machineId = String.format("%16x%16x\n", random.nextLong(), random.nextLong());
-            machineIdPath.createParents().writeUtf8File(machineId);
-            context.log(logger, "Wrote " + machineId + " to " + machineIdPath);
-        }
-
-        addMounts(context, command);
-
-        logger.info("Creating new container with args: " + command);
-        command.create();
-    }
-
-    private static void assertEqualIpAddresses(HostName hostName, Optional<? extends InetAddress> resolvedAddress,
-                                               Set<String> nrAddresses, IPVersion ipVersion) {
-        Optional<InetAddress> nrAddress = nrAddresses.stream()
-                .map(InetAddresses::forString)
-                .filter(ipVersion::match)
-                .findFirst();
-        if (resolvedAddress.equals(nrAddress)) return;
-
-        throw new ConvergenceException(String.format(
-                "IP address (%s) resolved from %s  does not match IP address (%s) in node-repo",
-                resolvedAddress.map(InetAddresses::toAddrString).orElse("[none]"), hostName,
-                nrAddress.map(InetAddresses::toAddrString).orElse("[none]")));
-    }
-
-    void addEtcHosts(ContainerData containerData,
-                     String hostname,
-                     Optional<? extends InetAddress> ipV4Local,
-                     Optional<? extends InetAddress> ipV6Local) {
-        // The default /etc/hosts in a Docker container contains one entry for the host,
-        // mapping the hostname to the Docker-assigned IPv4 address.
-        //
-        // When e.g. the cluster controller's ZooKeeper server starts, it binds the election
-        // port to the localhost's IP address as returned by InetAddress.getByName, backed by
-        // getaddrinfo(2), backed by that one entry in /etc/hosts. If the Docker container does
-        // not have a public IPv4 address, then other members of the ZooKeeper ensemble will not
-        // be able to connect.
-        //
-        // We will therefore explicitly manage the /etc/hosts file ourselves. Because of NPT,
-        // the IP addresses needs to be local.
-
-        StringBuilder etcHosts = new StringBuilder(
-                "# This file was generated by " + ContainerOperationsImpl.class.getName() + "\n" +
-                "127.0.0.1\tlocalhost\n" +
-                "::1\tlocalhost ip6-localhost ip6-loopback\n" +
-                "fe00::0\tip6-localnet\n" +
-                "ff00::0\tip6-mcastprefix\n" +
-                "ff02::1\tip6-allnodes\n" +
-                "ff02::2\tip6-allrouters\n");
-        ipV6Local.ifPresent(ipv6 -> etcHosts.append(ipv6.getHostAddress()).append('\t').append(hostname).append('\n'));
-        ipV4Local.ifPresent(ipv4 -> etcHosts.append(ipv4.getHostAddress()).append('\t').append(hostname).append('\n'));
-
-        containerData.addFile(fileSystem.getPath("/etc/hosts"), etcHosts.toString());
-    }
-
-    @Override
-    public void startContainer(NodeAgentContext context) {
-        context.log(logger, "Starting container");
-        containerEngine.startContainer(context.containerName());
-    }
-
-    @Override
-    public void removeContainer(NodeAgentContext context, Container container) {
-        if (container.state.isRunning()) {
-            context.log(logger, "Stopping container");
-            containerEngine.stopContainer(context.containerName());
-        }
-
-        context.log(logger, "Deleting container");
-        containerEngine.deleteContainer(context.containerName());
-    }
-
-    @Override
-    public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) {
-        containerEngine.updateContainer(context.containerName(), containerResources);
-    }
-
-    @Override
-    public Optional<Container> getContainer(NodeAgentContext context) {
-        return containerEngine.getContainer(context.containerName());
-    }
-
-    @Override
-    public boolean pullImageAsyncIfNeeded(TaskContext context, DockerImage dockerImage, RegistryCredentials registryCredentials) {
-        return containerEngine.pullImageAsyncIfNeeded(dockerImage, registryCredentials);
-    }
-
-    @Override
-    public ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command) {
-        return containerEngine.executeInContainerAsUser(context.containerName(), "root", OptionalLong.of(timeoutSeconds), command);
-    }
-
-    @Override
-    public ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, String... command) {
-        return containerEngine.executeInContainerAsUser(context.containerName(), "root", OptionalLong.empty(), command);
-    }
-
-    @Override
-    public CommandResult executeCommandInNetworkNamespace(NodeAgentContext context, String... command) {
-        int containerPid = containerEngine.getContainer(context.containerName())
-                .filter(container -> container.state.isRunning())
-                .orElseThrow(() -> new RuntimeException(
-                        "Found no running container named " + context.containerName().asString()))
-                .pid;
-
-        return terminal.newCommandLine(context)
-                .add("nsenter", String.format("--net=/proc/%d/ns/net", containerPid), "--")
-                .add(command)
-                .executeSilently();
-    }
-
-    @Override
-    public String resumeNode(NodeAgentContext context) {
-        return executeNodeCtlInContainer(context, "resume").getOutput();
-    }
-
-    @Override
-    public String suspendNode(NodeAgentContext context) {
-        return executeNodeCtlInContainer(context, "suspend").getOutput();
-    }
-
-    @Override
-    public String restartVespa(NodeAgentContext context) {
-        return executeNodeCtlInContainer(context, "restart-vespa").getOutput();
-    }
-
-    @Override
-    public String startServices(NodeAgentContext context) {
-        return executeNodeCtlInContainer(context, "start").getOutput();
-    }
-
-    @Override
-    public String stopServices(NodeAgentContext context) {
-        return executeNodeCtlInContainer(context, "stop").getOutput();
-    }
-
-    ProcessResult executeNodeCtlInContainer(NodeAgentContext context, String program) {
-        String[] command = new String[] {context.pathInNodeUnderVespaHome("bin/vespa-nodectl").toString(), program};
-        ProcessResult result = executeCommandInContainerAsRoot(context, command);
-
-        if (!result.isSuccess()) {
-            throw new RuntimeException("Container " + context.containerName().asString() +
-                    ": command " + Arrays.toString(command) + " failed: " + result);
-        }
-        return result;
-    }
-
-
-    @Override
-    public Optional<ContainerStats> getContainerStats(NodeAgentContext context) {
-        return containerEngine.getContainerStats(context.containerName());
-    }
-
-    private void addMounts(NodeAgentContext context, ContainerEngine.CreateContainerCommand command) {
-        var volumes = new VolumeHelper(context, command);
-
-        // Paths unique to each container
-        volumes.addPrivateVolumes(
-                ETC_MACHINE_ID,  // VESPA-18110, rotation of journal
-                "/etc/vespa/flags", // local file db, to use flags before connection to cfg is established
-                "/etc/yamas-agent", // metrics check configuration
-                "/opt/splunkforwarder/var/log",  // VESPA-14917, thin pool leakage
-                "/var/log",                      // VESPA-14917, thin pool leakage
-                "/var/log/journal",              // VESPA-18110, rotation of journal, must map exact path
-                "/var/spool/postfix/maildrop",
-
-                // Under VESPA_HOME in container
-                "logs/vespa",
-                "logs/ysar",
-                "tmp",
-                "var/crash", // core dumps
-                "var/container-data",
-                "var/db/vespa",
-                "var/jdisc_container",
-                "var/vespa",
-                "var/zookeeper");
-
-        if (context.nodeType() == NodeType.proxy) {
-            volumes.addPrivateVolumes("logs/nginx", "var/vespa-hosted/routing");
-        } else if (context.nodeType() == NodeType.tenant)
-            volumes.addPrivateVolumes("/var/lib/sia");
-
-        // Shared paths
-        if (isInfrastructureHost(context.nodeType()))
-            volumes.addSharedVolumeMap("/var/lib/sia", "/var/lib/sia");
-
-        boolean isMain = context.zone().getSystemName() == SystemName.cd || context.zone().getSystemName() == SystemName.main;
-        if (isMain && context.nodeType() == NodeType.tenant)
-            volumes.addSharedVolumeMap("/var/zpe", "var/zpe");
-    }
-
-    @Override
-    public boolean noManagedContainersRunning(TaskContext context) {
-        return containerEngine.noManagedContainersRunning(MANAGER_NAME);
-    }
-
-    @Override
-    public boolean retainManagedContainers(TaskContext context, Set<ContainerName> containerNames) {
-        return containerEngine.listManagedContainers(MANAGER_NAME).stream()
-                .filter(containerName -> ! containerNames.contains(containerName))
-                .peek(containerName -> {
-                    containerEngine.stopContainer(containerName);
-                    containerEngine.deleteContainer(containerName);
-                }).count() > 0;
-    }
-
-    @Override
-    public boolean deleteUnusedContainerImages(TaskContext context, List<DockerImage> excludes, Duration minImageAgeToDelete) {
-        return containerEngine.deleteUnusedDockerImages(excludes, minImageAgeToDelete);
-    }
-
-    /** Returns whether given nodeType is a Docker host for infrastructure nodes */
-    private static boolean isInfrastructureHost(NodeType nodeType) {
-        return nodeType == NodeType.config ||
-                nodeType == NodeType.proxy ||
-                nodeType == NodeType.controller;
-    }
-
-    private static class VolumeHelper {
-        private final NodeAgentContext context;
-        private final ContainerEngine.CreateContainerCommand command;
-
-        public VolumeHelper(NodeAgentContext context, ContainerEngine.CreateContainerCommand command) {
-            this.context = context;
-            this.command = command;
-        }
-
-        /**
-         * Resolve each path to an absolute relative the container's vespa home directory.
-         * Mounts the resulting path, under the container's storage directory as path in the container.
-         */
-        public void addPrivateVolumes(String... pathsInNode) {
-            Stream.of(pathsInNode).forEach(pathString -> {
-                Path absolutePathInNode = resolveNodePath(pathString);
-                Path pathOnHost = context.pathOnHostFromPathInNode(absolutePathInNode);
-                command.withVolume(pathOnHost, absolutePathInNode);
-            });
-        }
-
-        /**
-         * Mounts pathOnHost on the host as pathInNode in the container.  Use for paths that
-         * might be shared with other containers.
-         */
-        public void addSharedVolumeMap(String pathOnHost, String pathInNode) {
-            command.withSharedVolume(resolveNodePath(pathOnHost), resolveNodePath(pathInNode));
-        }
-
-        private Path resolveNodePath(String pathString) {
-            Path path = context.fileSystem().getPath(pathString);
-            return path.isAbsolute() ? path : context.pathInNodeUnderVespaHome(path);
-        }
-    }
-
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerOperationsImplTest.java
deleted file mode 100644
index e78d5bb754b..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerOperationsImplTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.docker;
-
-import com.google.common.net.InetAddresses;
-import com.yahoo.config.provision.DockerImage;
-import com.yahoo.vespa.hosted.dockerapi.Container;
-import com.yahoo.vespa.hosted.dockerapi.ContainerEngine;
-import com.yahoo.vespa.hosted.dockerapi.ContainerId;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
-import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
-import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses;
-import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock;
-import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
-import com.yahoo.vespa.test.file.TestFileSystem;
-import org.junit.Test;
-import org.mockito.InOrder;
-
-import java.net.InetAddress;
-import java.nio.file.FileSystem;
-import java.util.List;
-import java.util.Optional;
-import java.util.OptionalLong;
-import java.util.Set;
-
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.inOrder;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class ContainerOperationsImplTest {
-    private final ContainerEngine containerEngine = mock(ContainerEngine.class);
-    private final TestTerminal terminal = new TestTerminal();
-    private final IPAddresses ipAddresses = new IPAddressesMock();
-    private final FileSystem fileSystem = TestFileSystem.create();
-    private final ContainerOperationsImpl containerOperations = new ContainerOperationsImpl(
-            containerEngine, terminal, ipAddresses, fileSystem);
-
-    @Test
-    public void processResultFromNodeProgramWhenSuccess() {
-        final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build();
-        final ProcessResult actualResult = new ProcessResult(0, "output", "errors");
-
-        when(containerEngine.executeInContainerAsUser(any(), any(), any(), any()))
-                .thenReturn(actualResult); // output from node program
-
-        ProcessResult result = containerOperations.executeNodeCtlInContainer(context, "start");
-
-        final InOrder inOrder = inOrder(containerEngine);
-        inOrder.verify(containerEngine, times(1)).executeInContainerAsUser(
-                eq(context.containerName()),
-                eq("root"),
-                eq(OptionalLong.empty()),
-                eq("/opt/vespa/bin/vespa-nodectl"),
-                eq("start"));
-
-        assertThat(result, is(actualResult));
-    }
-
-    @Test(expected = RuntimeException.class)
-    public void processResultFromNodeProgramWhenNonZeroExitCode() {
-        final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build();
-        final ProcessResult actualResult = new ProcessResult(3, "output", "errors");
-
-        when(containerEngine.executeInContainerAsUser(any(), any(), any(), any()))
-                .thenReturn(actualResult); // output from node program
-
-        containerOperations.executeNodeCtlInContainer(context, "start");
-    }
-
-    @Test
-    public void runsCommandInNetworkNamespace() {
-        NodeAgentContext context = new NodeAgentContextImpl.Builder("container-42.domain.tld").build();
-        makeContainer("container-42", Container.State.RUNNING, 42);
-
-        terminal.expectCommand("nsenter --net=/proc/42/ns/net -- iptables -nvL 2>&1");
-
-        containerOperations.executeCommandInNetworkNamespace(context, "iptables", "-nvL");
-    }
-
-    private Container makeContainer(String name, Container.State state, int pid) {
-        final Container container = new Container(new ContainerId(name + "-id"), name + ".fqdn",
-                DockerImage.fromString("registry.example.com/mock"), null, new ContainerName(name), state, pid);
-        when(containerEngine.getContainer(eq(container.name))).thenReturn(Optional.of(container));
-        return container;
-    }
-
-    @Test
-    public void verifyEtcHosts() {
-        ContainerData containerData = mock(ContainerData.class);
-        String hostname = "hostname";
-        InetAddress ipV6Local = InetAddresses.forString("::1");
-        InetAddress ipV4Local = InetAddresses.forString("127.0.0.1");
-
-        containerOperations.addEtcHosts(containerData, hostname, Optional.empty(), Optional.of(ipV6Local));
-
-        verify(containerData, times(1)).addFile(
-                fileSystem.getPath("/etc/hosts"),
-                "# This file was generated by com.yahoo.vespa.hosted.node.admin.docker.ContainerOperationsImpl\n" +
-                        "127.0.0.1	localhost\n" +
-                        "::1	localhost ip6-localhost ip6-loopback\n" +
-                        "fe00::0	ip6-localnet\n" +
-                        "ff00::0	ip6-mcastprefix\n" +
-                        "ff02::1	ip6-allnodes\n" +
-                        "ff02::2	ip6-allrouters\n" +
-                        "0:0:0:0:0:0:0:1	hostname\n");
-
-        containerOperations.addEtcHosts(containerData, hostname, Optional.of(ipV4Local), Optional.of(ipV6Local));
-
-        verify(containerData, times(1)).addFile(
-                fileSystem.getPath("/etc/hosts"),
-                "# This file was generated by com.yahoo.vespa.hosted.node.admin.docker.ContainerOperationsImpl\n" +
-                        "127.0.0.1	localhost\n" +
-                        "::1	localhost ip6-localhost ip6-loopback\n" +
-                        "fe00::0	ip6-localnet\n" +
-                        "ff00::0	ip6-mcastprefix\n" +
-                        "ff02::1	ip6-allnodes\n" +
-                        "ff02::2	ip6-allrouters\n" +
-                        "0:0:0:0:0:0:0:1	hostname\n" +
-                        "127.0.0.1	hostname\n");
-    }
-
-    @Test
-    public void retainContainersTest() {
-        when(containerEngine.listManagedContainers(ContainerOperationsImpl.MANAGER_NAME))
-                .thenReturn(List.of(new ContainerName("cnt1"), new ContainerName("cnt2"), new ContainerName("cnt3")));
-        containerOperations.retainManagedContainers(any(), Set.of(new ContainerName("cnt2"), new ContainerName("cnt4")));
-
-        verify(containerEngine).stopContainer(eq(new ContainerName("cnt1")));
-        verify(containerEngine).deleteContainer(eq(new ContainerName("cnt1")));
-        verify(containerEngine).stopContainer(eq(new ContainerName("cnt3")));
-        verify(containerEngine).deleteContainer(eq(new ContainerName("cnt3")));
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java
new file mode 100644
index 00000000000..d187c8e5476
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java
@@ -0,0 +1,49 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.integration;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import org.junit.Test;
+
+import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+/**
+ * @author freva
+ */
+public class ContainerFailTest {
+
+    @Test
+    public void test() {
+        try (ContainerTester tester = new ContainerTester()) {
+            DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage");
+            ContainerName containerName = new ContainerName("host1");
+            String hostname = "host1.test.yahoo.com";
+            NodeSpec nodeSpec = NodeSpec.Builder
+                    .testSpec(hostname)
+                    .wantedDockerImage(dockerImage)
+                    .currentDockerImage(dockerImage)
+                    .build();
+            tester.addChildNodeRepositoryNode(nodeSpec);
+
+            NodeAgentContext context = new NodeAgentContextImpl.Builder(nodeSpec).build();
+
+            tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any(), any());
+            tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName));
+
+            tester.containerOperations.removeContainer(context, tester.containerOperations.getContainer(context).get());
+
+            tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(containerName), any());
+            tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any(), any());
+            tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName));
+
+            verify(tester.nodeRepository, never()).updateNodeAttributes(any(), any());
+        }
+    }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerOperationsMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerOperationsMock.java
new file mode 100644
index 00000000000..943bfb1dbaf
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerOperationsMock.java
@@ -0,0 +1,155 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.integration;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.dockerapi.Container;
+import com.yahoo.vespa.hosted.dockerapi.ContainerId;
+import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
+import com.yahoo.vespa.hosted.dockerapi.ContainerStats;
+import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
+import com.yahoo.vespa.hosted.dockerapi.RegistryCredentials;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.docker.ContainerOperations;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Mock with some simple logic
+ *
+ * @author freva
+ */
+public class ContainerOperationsMock implements ContainerOperations {
+
+    public static final ContainerId CONTAINER_ID = new ContainerId("af345");
+
+    private final Map<ContainerName, Container> containersByContainerName = new HashMap<>();
+    private final Object monitor = new Object();
+
+    @Override
+    public void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources) {
+        synchronized (monitor) {
+            containersByContainerName.put(context.containerName(),
+                                          new Container(CONTAINER_ID,
+                                                        context.hostname().value(),
+                                                        context.node().wantedDockerImage().get(),
+                                                        containerResources,
+                                                        context.containerName(),
+                                                        Container.State.CREATED,
+                                                        2));
+        }
+    }
+
+    @Override
+    public void startContainer(NodeAgentContext context) {
+        synchronized (monitor) {
+            Optional<Container> container = getContainer(context);
+            if (container.isEmpty())  throw new IllegalArgumentException("Cannot start non-existent container " + context.containerName());
+            containersByContainerName.put(context.containerName(), new Container(CONTAINER_ID,
+                                                                                 container.get().hostname,
+                                                                                 container.get().image,
+                                                                                 container.get().resources,
+                                                                                 container.get().name,
+                                                                                 Container.State.RUNNING,
+                                                                                 container.get().pid));
+        }
+    }
+
+    @Override
+    public void removeContainer(NodeAgentContext context, Container container) {
+        synchronized (monitor) {
+            containersByContainerName.remove(container.name);
+        }
+    }
+
+    @Override
+    public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) {
+        synchronized (monitor) {
+            Container container = containersByContainerName.get(context.containerName());
+            containersByContainerName.put(context.containerName(),
+                                          new Container(container.id(), container.hostname, container.image,
+                                                        containerResources, container.name, container.state, container.pid));
+        }
+    }
+
+    @Override
+    public Optional<Container> getContainer(NodeAgentContext context) {
+        synchronized (monitor) {
+            return Optional.ofNullable(containersByContainerName.get(context.containerName()));
+        }
+    }
+
+    @Override
+    public boolean pullImageAsyncIfNeeded(TaskContext context, DockerImage dockerImage, RegistryCredentials registryCredentials) {
+        return false;
+    }
+
+    @Override
+    public ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, String... command) {
+        return null;
+    }
+
+    @Override
+    public ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command) {
+        return null;
+    }
+
+    @Override
+    public CommandResult executeCommandInNetworkNamespace(NodeAgentContext context, String... command) {
+        return null;
+    }
+
+    @Override
+    public String resumeNode(NodeAgentContext context) {
+        return "";
+    }
+
+    @Override
+    public String suspendNode(NodeAgentContext context) {
+        return "";
+    }
+
+    @Override
+    public String restartVespa(NodeAgentContext context) {
+        return "";
+    }
+
+    @Override
+    public String startServices(NodeAgentContext context) {
+        return "";
+    }
+
+    @Override
+    public String stopServices(NodeAgentContext context) {
+        return "";
+    }
+
+    @Override
+    public Optional<ContainerStats> getContainerStats(NodeAgentContext context) {
+        return Optional.empty();
+    }
+
+    @Override
+    public boolean noManagedContainersRunning(TaskContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean retainManagedContainers(TaskContext context, Set<ContainerName> containerNames) {
+        return false;
+    }
+
+    @Override
+    public boolean deleteUnusedContainerImages(TaskContext context, List<DockerImage> excludes, Duration minImageAgeToDelete) {
+        return false;
+    }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java
index 9830bfce3a4..c5092e833b2 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java
@@ -1,26 +1,23 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
+package com.yahoo.vespa.hosted.node.admin.integration;
 
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
-import com.yahoo.vespa.hosted.dockerapi.ContainerEngine;
+import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.dockerapi.RegistryCredentials;
 import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
-import com.yahoo.vespa.hosted.node.admin.docker.ContainerOperations;
-import com.yahoo.vespa.hosted.node.admin.docker.ContainerOperationsImpl;
 import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl;
 import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock;
-import com.yahoo.vespa.hosted.node.admin.task.util.process.TerminalImpl;
-import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2;
 import com.yahoo.vespa.test.file.TestFileSystem;
 import org.mockito.InOrder;
 import org.mockito.Mockito;
@@ -35,6 +32,7 @@ import java.util.Optional;
 import java.util.logging.Logger;
 
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.timeout;
@@ -44,20 +42,20 @@ import static org.mockito.Mockito.when;
  * @author musum
  */
 // Need to deconstruct nodeAdminStateUpdater
-public class DockerTester implements AutoCloseable {
-    private static final Logger log = Logger.getLogger(DockerTester.class.getName());
+public class ContainerTester implements AutoCloseable {
+
+    private static final Logger log = Logger.getLogger(ContainerTester.class.getName());
     private static final Duration INTERVAL = Duration.ofMillis(10);
     private static final Path PATH_TO_VESPA_HOME = Paths.get("/opt/vespa");
-    static final String NODE_PROGRAM = PATH_TO_VESPA_HOME.resolve("bin/vespa-nodectl").toString();
     static final HostName HOST_HOSTNAME = HostName.from("host.test.yahoo.com");
 
     private final Thread loopThread;
 
-    final ContainerEngine containerEngine = spy(new ContainerEngineMock());
+    final ContainerOperationsMock containerOperations = spy(new ContainerOperationsMock());
     final NodeRepoMock nodeRepository = spy(new NodeRepoMock());
     final Orchestrator orchestrator = mock(Orchestrator.class);
     final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class);
-    final InOrder inOrder = Mockito.inOrder(containerEngine, nodeRepository, orchestrator, storageMaintainer);
+    final InOrder inOrder = Mockito.inOrder(containerOperations, nodeRepository, orchestrator, storageMaintainer);
     final InMemoryFlagSource flagSource = new InMemoryFlagSource();
 
     final NodeAdminStateUpdater nodeAdminStateUpdater;
@@ -67,7 +65,7 @@ public class DockerTester implements AutoCloseable {
     private volatile NodeAdminStateUpdater.State wantedState = NodeAdminStateUpdater.State.RESUMED;
 
 
-    DockerTester() {
+    ContainerTester() {
         when(storageMaintainer.diskUsageFor(any())).thenReturn(Optional.empty());
 
         IPAddressesMock ipAddresses = new IPAddressesMock();
@@ -75,15 +73,12 @@ public class DockerTester implements AutoCloseable {
         ipAddresses.addAddress(HOST_HOSTNAME.value(), "f000::");
         for (int i = 1; i < 4; i++) ipAddresses.addAddress("host" + i + ".test.yahoo.com", "f000::" + i);
 
-        TerminalImpl terminal = new TerminalImpl(command -> new TestChildProcess2(0, ""));
-
         NodeSpec hostSpec = NodeSpec.Builder.testSpec(HOST_HOSTNAME.value()).type(NodeType.host).build();
         nodeRepository.updateNodeRepositoryNode(hostSpec);
 
         Clock clock = Clock.systemUTC();
         Metrics metrics = new Metrics();
         FileSystem fileSystem = TestFileSystem.create();
-        ContainerOperations containerOperations = new ContainerOperationsImpl(containerEngine, terminal, ipAddresses, fileSystem);
 
         NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> new NodeAgentImpl(
                 contextSupplier, nodeRepository, orchestrator, containerOperations, () -> RegistryCredentials.none,
@@ -129,6 +124,10 @@ public class DockerTester implements AutoCloseable {
         return inOrder.verify(t, timeout(5000));
     }
 
+    public static NodeAgentContext containerMatcher(ContainerName containerName) {
+        return argThat((ctx) -> ctx.containerName().equals(containerName));
+    }
+
     @Override
     public void close() {
         // First, stop NodeAdmin and all the NodeAgents
@@ -143,4 +142,5 @@ public class DockerTester implements AutoCloseable {
             }
         } while (loopThread.isAlive());
     }
+
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java
index 84089d94d45..78e9dc78a0f 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java
@@ -1,11 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
+package com.yahoo.vespa.hosted.node.admin.integration;
 
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import org.junit.Test;
 
 import static org.mockito.ArgumentMatchers.any;
@@ -15,11 +15,11 @@ import static org.mockito.ArgumentMatchers.eq;
 /**
  * @author freva
  */
-public class MultiDockerTest {
+public class MultiContainerTest {
 
     @Test
     public void test() {
-        try (DockerTester tester = new DockerTester()) {
+        try (ContainerTester tester = new ContainerTester()) {
             DockerImage image1 = DockerImage.fromString("registry.example.com/image1");
             addAndWaitForNode(tester, "host1.test.yahoo.com", image1);
             NodeSpec nodeSpec2 = addAndWaitForNode(
@@ -27,26 +27,30 @@ public class MultiDockerTest {
 
             tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(nodeSpec2.hostname(), NodeState.dirty).build());
 
-            tester.inOrder(tester.containerEngine).deleteContainer(eq(new ContainerName("host2")));
+            ContainerName host2 = new ContainerName("host2");
+            tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(host2), any());
             tester.inOrder(tester.storageMaintainer).archiveNodeStorage(
-                    argThat(context -> context.containerName().equals(new ContainerName("host2"))));
+                    argThat(context -> context.containerName().equals(host2)));
             tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.hostname()), eq(NodeState.ready));
 
             addAndWaitForNode(tester, "host3.test.yahoo.com", image1);
         }
     }
 
-    private NodeSpec addAndWaitForNode(DockerTester tester, String hostName, DockerImage dockerImage) {
+    private NodeAgentContext containerMatcher(ContainerName containerName) {
+        return argThat((ctx) -> ctx.containerName().equals(containerName));
+    }
+
+    private NodeSpec addAndWaitForNode(ContainerTester tester, String hostName, DockerImage dockerImage) {
         NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostName).wantedDockerImage(dockerImage).build();
         tester.addChildNodeRepositoryNode(nodeSpec);
 
         ContainerName containerName = ContainerName.fromHostname(hostName);
-        tester.inOrder(tester.containerEngine).createContainerCommand(eq(dockerImage), eq(containerName));
-        tester.inOrder(tester.containerEngine).executeInContainerAsUser(
-                eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume"));
-        tester.inOrder(tester.nodeRepository).updateNodeAttributes(eq(hostName),
-                eq(new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion())));
+        tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any(), any());
+        tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName));
+        tester.inOrder(tester.nodeRepository).updateNodeAttributes(eq(hostName), any());
 
         return nodeSpec;
     }
+
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java
index 80069b38748..83154723266 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java
@@ -1,5 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
+package com.yahoo.vespa.hosted.node.admin.integration;
 
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.AddNode;
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java
index d1bc6fc0daa..c77c3034142 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java
@@ -1,5 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
+package com.yahoo.vespa.hosted.node.admin.integration;
 
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
@@ -8,11 +8,11 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater;
 import org.junit.Test;
 
 import java.util.List;
-import java.util.OptionalLong;
 
-import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.HOST_HOSTNAME;
-import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.NODE_PROGRAM;
+import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.HOST_HOSTNAME;
+import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 
 /**
@@ -27,10 +27,11 @@ public class RebootTest {
 
     @Test
     public void test() {
-        try (DockerTester tester = new DockerTester()) {
+        try (ContainerTester tester = new ContainerTester()) {
             tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build());
 
-            tester.inOrder(tester.containerEngine).createContainerCommand(eq(dockerImage), eq(new ContainerName("host1")));
+            ContainerName host1 = new ContainerName("host1");
+            tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any(), any());
 
             try {
                 tester.setWantedState(NodeAdminStateUpdater.State.SUSPENDED);
@@ -38,9 +39,9 @@ public class RebootTest {
 
             tester.inOrder(tester.orchestrator).suspend(
                     eq(HOST_HOSTNAME.value()), eq(List.of(hostname, HOST_HOSTNAME.value())));
-            tester.inOrder(tester.containerEngine).executeInContainerAsUser(
-                    eq(new ContainerName("host1")), eq("root"), eq(OptionalLong.empty()), eq(NODE_PROGRAM), eq("stop"));
+            tester.inOrder(tester.containerOperations).stopServices(containerMatcher(host1));
             assertTrue(tester.nodeAdmin.setFrozen(true));
         }
     }
+
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java
index 645a89cc20e..319623e69de 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java
@@ -1,5 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
+package com.yahoo.vespa.hosted.node.admin.integration;
 
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
@@ -7,7 +7,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttribu
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import org.junit.Test;
 
-import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.NODE_PROGRAM;
+import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 
@@ -20,13 +20,15 @@ public class RestartTest {
 
     @Test
     public void test() {
-        try (DockerTester tester = new DockerTester()) {
+        try (ContainerTester tester = new ContainerTester()) {
             String hostname = "host1.test.yahoo.com";
             DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage:1.2.3");
 
-            tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build());
+            NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build();
+            tester.addChildNodeRepositoryNode(nodeSpec);
 
-            tester.inOrder(tester.containerEngine).createContainerCommand(eq(dockerImage), eq(new ContainerName("host1")));
+            ContainerName host1 = new ContainerName("host1");
+            tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any(), any());
             tester.inOrder(tester.nodeRepository).updateNodeAttributes(
                     eq(hostname), eq(new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion())));
 
@@ -35,10 +37,10 @@ public class RestartTest {
                     .wantedRestartGeneration(2).build());
 
             tester.inOrder(tester.orchestrator).suspend(eq(hostname));
-            tester.inOrder(tester.containerEngine).executeInContainerAsUser(
-                    eq(new ContainerName("host1")), any(), any(), eq(NODE_PROGRAM), eq("restart-vespa"));
+            tester.inOrder(tester.containerOperations).restartVespa(containerMatcher(host1));
             tester.inOrder(tester.nodeRepository).updateNodeAttributes(
                     eq(hostname), eq(new NodeAttributes().withRestartGeneration(2)));
         }
     }
+
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ContainerEngineMock.java
deleted file mode 100644
index a856c9d9b8d..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ContainerEngineMock.java
+++ /dev/null
@@ -1,212 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
-
-import com.yahoo.config.provision.DockerImage;
-import com.yahoo.vespa.hosted.dockerapi.Container;
-import com.yahoo.vespa.hosted.dockerapi.ContainerEngine;
-import com.yahoo.vespa.hosted.dockerapi.ContainerId;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
-import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
-import com.yahoo.vespa.hosted.dockerapi.ContainerStats;
-import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
-import com.yahoo.vespa.hosted.dockerapi.RegistryCredentials;
-
-import java.net.InetAddress;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.OptionalLong;
-
-/**
- * Mock with some simple logic
- *
- * @author freva
- */
-public class ContainerEngineMock implements ContainerEngine {
-    public static final ContainerId CONTAINER_ID = new ContainerId("af345");
-
-    private final Map<ContainerName, Container> containersByContainerName = new HashMap<>();
-    private static final Object monitor = new Object();
-
-    @Override
-    public CreateContainerCommand createContainerCommand(DockerImage dockerImage, ContainerName containerName) {
-        return new StartContainerCommandMock(CONTAINER_ID, dockerImage, containerName);
-    }
-
-    @Override
-    public Optional<ContainerStats> getContainerStats(ContainerName containerName) {
-        return Optional.empty();
-    }
-
-    @Override
-    public void startContainer(ContainerName containerName) {
-
-    }
-
-    @Override
-    public void stopContainer(ContainerName containerName) {
-        synchronized (monitor) {
-            Container container = containersByContainerName.get(containerName);
-            containersByContainerName.put(containerName,
-                            new Container(container.id(), container.hostname, container.image, container.resources, container.name, Container.State.EXITED, 0));
-        }
-    }
-
-    @Override
-    public void deleteContainer(ContainerName containerName) {
-        synchronized (monitor) {
-            containersByContainerName.remove(containerName);
-        }
-    }
-
-    @Override
-    public void updateContainer(ContainerName containerName, ContainerResources containerResources) {
-        synchronized (monitor) {
-            Container container = containersByContainerName.get(containerName);
-            containersByContainerName.put(containerName,
-                    new Container(container.id(), container.hostname, container.image, containerResources, container.name, container.state, container.pid));
-        }
-    }
-
-    @Override
-    public Optional<Container> getContainer(ContainerName containerName) {
-        synchronized (monitor) {
-            return Optional.ofNullable(containersByContainerName.get(containerName));
-        }
-    }
-
-    @Override
-    public boolean pullImageAsyncIfNeeded(DockerImage image, RegistryCredentials credentials) {
-        synchronized (monitor) {
-            return false;
-        }
-    }
-
-    @Override
-    public boolean deleteUnusedDockerImages(List<DockerImage> excludes, Duration minImageAgeToDelete) {
-        return false;
-    }
-
-    @Override
-    public ProcessResult executeInContainerAsUser(ContainerName containerName, String user, OptionalLong timeout, String... args) {
-        return new ProcessResult(0, "", "");
-    }
-
-    @Override
-    public boolean noManagedContainersRunning(String manager) {
-        return false;
-    }
-
-    @Override
-    public List<ContainerName> listManagedContainers(String manager) {
-        return List.copyOf(containersByContainerName.keySet());
-    }
-
-    public class StartContainerCommandMock implements CreateContainerCommand {
-
-        private final ContainerId containerId;
-        private final DockerImage dockerImage;
-        private final ContainerName containerName;
-        private String hostName;
-        private ContainerResources containerResources;
-
-        public StartContainerCommandMock(ContainerId containerId, DockerImage dockerImage, ContainerName containerName) {
-            this.containerId = containerId;
-            this.dockerImage = dockerImage;
-            this.containerName = containerName;
-        }
-
-        @Override
-        public CreateContainerCommand withHostName(String hostname) {
-            this.hostName = hostname;
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withResources(ContainerResources containerResources) {
-            this.containerResources = containerResources;
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withLabel(String name, String value) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withEnvironment(String name, String value) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withVolume(Path path, Path volumePath) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withSharedVolume(Path path, Path volumePath) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withNetworkMode(String mode) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withIpAddress(InetAddress address) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withUlimit(String name, int softLimit, int hardLimit) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withEntrypoint(String... entrypoint) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withManagedBy(String manager) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withAddCapability(String capabilityName) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withDropCapability(String capabilityName) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withSecurityOpt(String securityOpt) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withDnsOption(String dnsOption) {
-            return this;
-        }
-
-        @Override
-        public CreateContainerCommand withPrivileged(boolean privileged) {
-            return this;
-        }
-
-        @Override
-        public void create() {
-            synchronized (monitor) {
-                containersByContainerName.put(
-                        containerName, new Container(containerId, hostName, dockerImage, containerResources, containerName, Container.State.RUNNING, 2));
-            }
-        }
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
deleted file mode 100644
index 88932f221a6..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integrationTests;
-
-import com.yahoo.config.provision.DockerImage;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import org.junit.Test;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-
-/**
- * @author freva
- */
-public class DockerFailTest {
-
-    @Test
-    public void dockerFailTest() {
-        try (DockerTester tester = new DockerTester()) {
-            final DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage");
-            final ContainerName containerName = new ContainerName("host1");
-            final String hostname = "host1.test.yahoo.com";
-            tester.addChildNodeRepositoryNode(NodeSpec.Builder
-                    .testSpec(hostname)
-                    .wantedDockerImage(dockerImage)
-                    .currentDockerImage(dockerImage)
-                    .build());
-
-            tester.inOrder(tester.containerEngine).createContainerCommand(eq(dockerImage), eq(containerName));
-            tester.inOrder(tester.containerEngine).executeInContainerAsUser(
-                    eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume"));
-
-            tester.containerEngine.deleteContainer(new ContainerName("host1"));
-
-            tester.inOrder(tester.containerEngine).deleteContainer(eq(containerName));
-            tester.inOrder(tester.containerEngine).createContainerCommand(eq(dockerImage), eq(containerName));
-            tester.inOrder(tester.containerEngine).executeInContainerAsUser(
-                    eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume"));
-
-            verify(tester.nodeRepository, never()).updateNodeAttributes(any(), any());
-        }
-    }
-}