Container cleanup

This PR puts all container-related code, which is independent of the container
engine implementation, in `com.yahoo.vespa.hosted.node.admin.container`.

* All operations on containers still pass through `ContainerOperations`
* `ContainerOperations` is no longer an interface
* `ContainerEngine` is the interface for a container engine/runtime (e.g.
Podman)
* Some code was migrated from internal repo. This is the reason for the
large-ish diff
* Fixes problematic circular dependencies

10 files changed, 713 insertions, 194 deletions

diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java
new file mode 100644
index 00000000000..1d077449ed6
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java
@@ -0,0 +1,206 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.image.Image;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.stream.Collectors;
+
+/**
+ * @author mpolden
+ */
+public class ContainerEngineMock implements ContainerEngine {
+
+    private final Map<ContainerName, Container> containers = new ConcurrentHashMap<>();
+    private final Map<String, ImageDownload> images = new ConcurrentHashMap<>();
+    private boolean asyncImageDownload = false;
+
+    public ContainerEngineMock asyncImageDownload(boolean enabled) {
+        this.asyncImageDownload = enabled;
+        return this;
+    }
+
+    public ContainerEngineMock completeDownloadOf(DockerImage image) {
+        String imageId = image.asString();
+        ImageDownload download;
+        while ((download = images.get(imageId)) == null);
+        download.complete();
+        return this;
+    }
+
+    public ContainerEngineMock setImages(List<Image> images) {
+        this.images.clear();
+        for (var image : images) {
+            ImageDownload imageDownload = new ImageDownload(image);
+            imageDownload.complete();
+            this.images.put(image.id(), imageDownload);
+        }
+        return this;
+    }
+
+    public ContainerEngineMock addContainers(List<Container> containers) {
+        for (var container : containers) {
+            if (this.containers.containsKey(container.name())) {
+                throw new IllegalArgumentException("Container " + container.name() + " already exists");
+            }
+            this.containers.put(container.name(), container);
+        }
+        return this;
+    }
+
+    public ContainerEngineMock addContainer(Container container) {
+        return addContainers(List.of(container));
+    }
+
+    @Override
+    public void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources) {
+        addContainer(createContainer(context, PartialContainer.State.created, containerResources));
+    }
+
+    @Override
+    public void startContainer(NodeAgentContext context) {
+        Container container = requireContainer(context.containerName(), PartialContainer.State.created);
+        Container newContainer = createContainer(context, PartialContainer.State.running, container.resources());
+        containers.put(newContainer.name(), newContainer);
+    }
+
+    @Override
+    public void removeContainer(TaskContext context, PartialContainer container) {
+        requireContainer(container.name());
+        containers.remove(container.name());
+    }
+
+    @Override
+    public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) {
+        Container container = requireContainer(context.containerName());
+        containers.put(container.name(), new Container(containerId, container.name(), container.state(),
+                                                       container.imageId(), container.image(),
+                                                       container.labels(), container.pid(),
+                                                       container.conmonPid(), container.hostname(),
+                                                       containerResources, container.networks(),
+                                                       container.managed()));
+    }
+
+    @Override
+    public Optional<Container> getContainer(NodeAgentContext context) {
+        return Optional.ofNullable(containers.get(context.containerName()));
+    }
+
+    @Override
+    public List<PartialContainer> listContainers(TaskContext context) {
+        return List.copyOf(containers.values());
+    }
+
+    @Override
+    public String networkInterface(NodeAgentContext context) {
+        return "eth0";
+    }
+
+    @Override
+    public CommandResult executeAsRoot(NodeAgentContext context, Duration timeout, String... command) {
+        return new CommandResult(null, 0, "");
+    }
+
+    @Override
+    public CommandResult executeInNetworkNamespace(NodeAgentContext context, String... command) {
+        return new CommandResult(null, 0, "");
+    }
+
+    @Override
+    public void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials) {
+        String imageId = image.asString();
+        ImageDownload imageDownload = images.computeIfAbsent(imageId, (ignored) -> new ImageDownload(new Image(imageId, Optional.empty(), List.of(imageId))));
+        if (!asyncImageDownload) {
+            imageDownload.complete();
+        }
+        imageDownload.awaitCompletion();
+    }
+
+    @Override
+    public boolean hasImage(TaskContext context, DockerImage image) {
+        ImageDownload download = images.get(image.asString());
+        return download != null && download.isComplete();
+    }
+
+    @Override
+    public void removeImage(TaskContext context, String id) {
+        images.remove(id);
+    }
+
+    @Override
+    public List<Image> listImages(TaskContext context) {
+        return images.values().stream()
+                     .filter(ImageDownload::isComplete)
+                     .map(ImageDownload::image)
+                     .collect(Collectors.toUnmodifiableList());
+    }
+
+    private Container requireContainer(ContainerName name) {
+        return requireContainer(name, null);
+    }
+
+    private Container requireContainer(ContainerName name, PartialContainer.State wantedState) {
+        Container container = containers.get(name);
+        if (container == null) throw new IllegalArgumentException("No such container: " + name);
+        if (wantedState != null && container.state() != wantedState) throw new IllegalArgumentException("Container is " + container.state() + ", wanted " + wantedState);
+        return container;
+    }
+
+    public Container createContainer(NodeAgentContext context, PartialContainer.State state, ContainerResources containerResources) {
+        return new Container(new ContainerId("id-of-" + context.containerName()),
+                             context.containerName(),
+                             state,
+                             "image-id",
+                             context.node().wantedDockerImage().get(),
+                             Map.of(),
+                             41,
+                             42,
+                             context.hostname().value(),
+                             containerResources,
+                             List.of(),
+                             true);
+    }
+
+    private static class ImageDownload {
+
+        private final Image image;
+        private final CountDownLatch done = new CountDownLatch(1);
+
+        ImageDownload(Image image) {
+            this.image = Objects.requireNonNull(image);
+        }
+
+        Image image() {
+            return image;
+        }
+
+        boolean isComplete() {
+            return done.getCount() == 0;
+        }
+
+        void complete() {
+            done.countDown();
+        }
+
+        void awaitCompletion() {
+            try {
+                done.await();
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+    }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java
new file mode 100644
index 00000000000..666c5fb31f9
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java
@@ -0,0 +1,64 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class ContainerOperationsTest {
+
+    private final TestTaskContext context = new TestTaskContext();
+    private final ContainerEngineMock containerEngine = new ContainerEngineMock();
+    private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, TestFileSystem.create());
+
+    @Test
+    public void no_managed_containers_running() {
+        Container c1 = createContainer("c1", true);
+        Container c2 = createContainer("c2", false);
+
+        containerEngine.addContainer(c1);
+        assertFalse(containerOperations.noManagedContainersRunning(context));
+
+        containerEngine.removeContainer(context, c1);
+        assertTrue(containerOperations.noManagedContainersRunning(context));
+
+        containerEngine.addContainer(c2);
+        assertTrue(containerOperations.noManagedContainersRunning(context));
+    }
+
+    @Test
+    public void retain_managed_containers() {
+        Container c1 = createContainer("c1", true);
+        Container c2 = createContainer("c2", true);
+        Container c3 = createContainer("c3", false);
+        containerEngine.addContainers(List.of(c1, c2, c3));
+
+        assertEquals(3, containerEngine.listContainers(context).size());
+        containerOperations.retainManagedContainers(context, Set.of(c1.name()));
+
+        assertEquals(List.of(c1.name(), c3.name()), containerEngine.listContainers(context).stream()
+                                                                   .map(PartialContainer::name)
+                                                                   .sorted()
+                                                                   .collect(Collectors.toList()));
+    }
+
+    private Container createContainer(String name, boolean managed) {
+        return new Container(new ContainerId("id-of-" + name), new ContainerName(name), PartialContainer.State.running,
+                             "image-id", DockerImage.EMPTY, Map.of(), 42, 43, name,
+                             ContainerResources.UNLIMITED, List.of(), managed);
+    }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java
new file mode 100644
index 00000000000..0bb7aee8e0a
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java
@@ -0,0 +1,151 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class ContainerStatsCollectorTest {
+
+    private final FileSystem fileSystem = TestFileSystem.create();
+
+    @Test
+    public void collect() throws Exception {
+        ContainerStatsCollector collector = new ContainerStatsCollector(fileSystem);
+        ContainerId containerId = new ContainerId("id1");
+        int containerPid = 42;
+        assertTrue("No stats found", collector.collect(containerId, containerPid, "eth0").isEmpty());
+
+        mockMemoryStats(containerId);
+        mockCpuStats(containerId);
+        mockNetworkStats(containerPid);
+
+        Optional<ContainerStats> stats = collector.collect(containerId, containerPid, "eth0");
+        assertTrue(stats.isPresent());
+        assertEquals(new ContainerStats.CpuStats(24, 6049374780000000L, 691675615472L,
+                                                 262190000000L, 3L, 1L, 2L),
+                     stats.get().getCpuStats());
+        assertEquals(new ContainerStats.MemoryStats(470790144L, 1228017664L, 2147483648L),
+                     stats.get().getMemoryStats());
+        assertEquals(Map.of("eth0", new ContainerStats.NetworkStats(22280813L, 4L, 3L,
+                                                                    19859383L, 6L, 5L)),
+                     stats.get().getNetworks());
+    }
+
+    private void mockNetworkStats(int pid) throws IOException {
+        Path dev = fileSystem.getPath("/proc/" + pid + "/net/dev");
+        Files.createDirectories(dev.getParent());
+        Files.writeString(dev, "Inter-|   Receive                                                |  Transmit\n" +
+                               " face |bytes    packets errs drop fifo frame compressed multicast|bytes    packets errs drop fifo colls carrier compressed\n" +
+                               "    lo: 36289258  149700    0    0    0     0          0         0 36289258  149700    0    0    0     0       0          0\n" +
+                               "  eth0: 22280813  118083    3    4    0     0          0         0 19859383  115415    5    6    0     0       0          0\n");
+    }
+
+    private void mockMemoryStats(ContainerId containerId) throws IOException {
+        Path root = fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-" + containerId + ".scope");
+        Files.createDirectories(root);
+
+        Files.writeString(root.resolve("memory.limit_in_bytes"), "2147483648\n");
+        Files.writeString(root.resolve("memory.usage_in_bytes"), "1228017664\n");
+        Files.writeString(root.resolve("memory.stat"), "cache 470790144\n" +
+                                                       "rss 698699776\n" +
+                                                       "rss_huge 526385152\n" +
+                                                       "shmem 0\n" +
+                                                       "mapped_file 811008\n" +
+                                                       "dirty 405504\n" +
+                                                       "writeback 0\n" +
+                                                       "swap 0\n" +
+                                                       "pgpgin 6938085\n" +
+                                                       "pgpgout 6780573\n" +
+                                                       "pgfault 14343186\n" +
+                                                       "pgmajfault 0\n" +
+                                                       "inactive_anon 0\n" +
+                                                       "active_anon 699289600\n" +
+                                                       "inactive_file 455516160\n" +
+                                                       "active_file 13787136\n" +
+                                                       "unevictable 0\n" +
+                                                       "hierarchical_memory_limit 2147483648\n" +
+                                                       "hierarchical_memsw_limit 4294967296\n" +
+                                                       "total_cache 470790144\n" +
+                                                       "total_rss 698699776\n" +
+                                                       "total_rss_huge 526385152\n" +
+                                                       "total_shmem 0\n" +
+                                                       "total_mapped_file 811008\n" +
+                                                       "total_dirty 405504\n" +
+                                                       "total_writeback 0\n" +
+                                                       "total_swap 0\n" +
+                                                       "total_pgpgin 6938085\n" +
+                                                       "total_pgpgout 6780573\n" +
+                                                       "total_pgfault 14343186\n" +
+                                                       "total_pgmajfault 0\n" +
+                                                       "total_inactive_anon 0\n" +
+                                                       "total_active_anon 699289600\n" +
+                                                       "total_inactive_file 455516160\n" +
+                                                       "total_active_file 13787136\n" +
+                                                       "total_unevictable 0\n");
+    }
+
+    private void mockCpuStats(ContainerId containerId) throws IOException {
+        Path root = fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-" + containerId + ".scope");
+        Path proc = fileSystem.getPath("/proc");
+        Files.createDirectories(root);
+        Files.createDirectories(proc);
+        Files.writeString(root.resolve("cpu.stat"), "nr_periods 1\n" +
+                                                    "nr_throttled 2\n" +
+                                                    "throttled_time 3\n");
+        Files.writeString(root.resolve("cpuacct.usage_percpu"), "25801608855 22529436415 25293652376 26212081533 " +
+                                                                "27545883290 25357818592 33464821448 32568003867 " +
+                                                                "28916742231 31771772292 34418037242 38417072233 " +
+                                                                "26069101127 24568838237 23683334366 26824607997 " +
+                                                                "24289870206 22249389818 32683986446 32444831154 " +
+                                                                "30488394217 26840956322 31633747261 30838696584\n");
+        Files.writeString(root.resolve("cpuacct.usage"), "691675615472\n");
+        Files.writeString(root.resolve("cpuacct.stat"), "user 40900\n" +
+                                                        "system 26219\n");
+        Files.writeString(proc.resolve("stat"), "cpu  7991366 978222 2346238 565556517 1935450 25514479 615206 0 0 0\n" +
+                                                "cpu0 387906 61529 99088 23516506 42258 1063359 29882 0 0 0\n" +
+                                                "cpu1 271253 49383 86149 23655234 41703 1061416 31885 0 0 0\n" +
+                                                "cpu2 349420 50987 93560 23571695 59437 1051977 24461 0 0 0\n" +
+                                                "cpu3 328107 50628 93406 23605135 44378 1048549 30199 0 0 0\n" +
+                                                "cpu4 267474 50404 99253 23606041 113094 1038572 26494 0 0 0\n" +
+                                                "cpu5 309584 50677 94284 23550372 132616 1033661 29436 0 0 0\n" +
+                                                "cpu6 477926 56888 121251 23367023 83121 1074930 28818 0 0 0\n" +
+                                                "cpu7 335335 29350 106130 23551107 95606 1066394 26156 0 0 0\n" +
+                                                "cpu8 323678 28629 99171 23586501 82183 1064708 25403 0 0 0\n" +
+                                                "cpu9 329805 27516 98538 23579458 89235 1061561 25140 0 0 0\n" +
+                                                "cpu10 291536 26455 93934 23642345 81282 1049736 25228 0 0 0\n" +
+                                                "cpu11 271103 25302 90630 23663641 85711 1048781 24291 0 0 0\n" +
+                                                "cpu12 323634 63392 100406 23465340 132684 1089157 28319 0 0 0\n" +
+                                                "cpu13 348085 49568 100772 23490388 114190 1079474 20948 0 0 0\n" +
+                                                "cpu14 310712 51208 90461 23547980 101601 1071940 26712 0 0 0\n" +
+                                                "cpu15 360405 52754 94620 23524878 79851 1062050 26836 0 0 0\n" +
+                                                "cpu16 367893 52141 98074 23541314 57500 1058968 25242 0 0 0\n" +
+                                                "cpu17 412756 51486 101592 23515056 47653 1044874 27467 0 0 0\n" +
+                                                "cpu18 287307 25478 106011 23599505 79848 1089812 23160 0 0 0\n" +
+                                                "cpu19 275001 24421 98338 23628694 79675 1084074 22083 0 0 0\n" +
+                                                "cpu20 288038 24805 94432 23629908 74735 1078501 21915 0 0 0\n" +
+                                                "cpu21 295373 25017 91344 23628585 75282 1071019 22026 0 0 0\n" +
+                                                "cpu22 326739 25588 90385 23608217 69186 1068494 21108 0 0 0\n" +
+                                                "cpu23 452284 24602 104397 23481583 72612 1052462 21985 0 0 0\n" +
+                                                "intr 6645352968 64 0 0 0 1481 0 0 0 1 0 0 0 0 0 0 0 39 0 0 0 0 0 0 37 0 0 0 0 0 0 0 0 4334106 1 6949071 5814662 5415344 6939471 6961483 6358810 5271953 6718644 0 126114 126114 126114 126114 126114 126114 126114 126114 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n" +
+                                                "ctxt 2495530303\n" +
+                                                "btime 1611928223\n" +
+                                                "processes 4839481\n" +
+                                                "procs_running 4\n" +
+                                                "procs_blocked 0\n" +
+                                                "softirq 2202631388 4 20504999 46734 54405637 4330276 0 6951 1664780312 10130 458546345\n");
+    }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ProcessResultTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ProcessResultTest.java
deleted file mode 100644
index f7b832bd566..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ProcessResultTest.java
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.container;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class ProcessResultTest {
-    @Test
-    public void testBasicProperties() {
-        ProcessResult processResult = new ProcessResult(0, "foo", "bar");
-        assertEquals(0, processResult.getExitStatus());
-        assertEquals("foo", processResult.getOutput());
-        assertTrue(processResult.isSuccess());
-    }
-
-    @Test
-    public void testSuccessFails() {
-        ProcessResult processResult = new ProcessResult(1, "foo", "bar");
-        assertFalse(processResult.isSuccess());
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java
new file mode 100644
index 00000000000..f50f2b8b053
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java
@@ -0,0 +1,34 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.image;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock;
+import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class ContainerImageDownloaderTest {
+
+    @Test(timeout = 5_000)
+    public void test_download() {
+        ContainerEngineMock podman = new ContainerEngineMock().asyncImageDownload(true);
+        ContainerImageDownloader downloader = new ContainerImageDownloader(podman);
+        TaskContext context = new TestTaskContext();
+        DockerImage image = DockerImage.fromString("registry.example.com/vespa:7.42");
+
+        assertFalse("Download started", downloader.get(context, image, RegistryCredentials.none));
+        assertFalse("Download pending", downloader.get(context, image, RegistryCredentials.none));
+        podman.completeDownloadOf(image);
+        boolean downloadCompleted;
+        while (!(downloadCompleted = downloader.get(context, image, RegistryCredentials.none)));
+        assertTrue("Download completed", downloadCompleted);
+    }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java
new file mode 100644
index 00000000000..f6d941c4299
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java
@@ -0,0 +1,238 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.image;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.Container;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerId;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerResources;
+import com.yahoo.vespa.hosted.node.admin.container.image.ContainerImagePruner;
+import com.yahoo.vespa.hosted.node.admin.container.image.Image;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author freva
+ * @author mpolden
+ */
+public class ContainerImagePrunerTest {
+
+    private final Tester tester = new Tester();
+
+    @Test
+    public void noImagesMeansNoUnusedImages() {
+        tester.withExistingImages()
+              .expectDeletedImages();
+    }
+
+    @Test
+    public void singleImageWithoutContainersIsUnused() {
+        tester.withExistingImages(image("image-1"))
+              // Even though nothing is using the image, we will keep it for at least 1h
+              .expectDeletedImagesAfterMinutes(0)
+              .expectDeletedImagesAfterMinutes(30)
+              .expectDeletedImagesAfterMinutes(30, "image-1");
+    }
+
+    @Test
+    public void singleImageWithContainerIsUsed() {
+        tester.withExistingImages(image("image-1"))
+              .withExistingContainers(container("container-1", "image-1"))
+              .expectDeletedImages();
+    }
+
+
+    @Test
+    public void multipleUnusedImagesAreIdentified() {
+        tester.withExistingImages(image("image-1"), image("image-2"))
+              .expectDeletedImages("image-1", "image-2");
+    }
+
+
+    @Test
+    public void multipleUnusedLeavesAreIdentified() {
+        tester.withExistingImages(image("parent-image"),
+                                  image("image-1", "parent-image"),
+                                  image("image-2", "parent-image"))
+              .expectDeletedImages("image-1", "image-2", "parent-image");
+    }
+
+
+    @Test
+    public void unusedLeafWithUsedSiblingIsIdentified() {
+        tester.withExistingImages(image("parent-image"),
+                                  image("image-1", "parent-image", "latest"),
+                                  image("image-2", "parent-image", "1.24"))
+              .withExistingContainers(container("vespa-node-1", "image-1"))
+              .expectDeletedImages("1.24"); // Deleting the only tag will delete the image
+    }
+
+
+    @Test
+    public void unusedImagesWithMultipleTags() {
+        tester.withExistingImages(image("parent-image"),
+                                  image("image-1", "parent-image", "vespa-6", "vespa-6.28", "vespa:latest"))
+              .expectDeletedImages("vespa-6", "vespa-6.28", "vespa:latest", "parent-image");
+    }
+
+
+    @Test
+    public void unusedImagesWithMultipleUntagged() {
+        tester.withExistingImages(image("image1", null, "<none>:<none>"),
+                                  image("image2", null, "<none>:<none>"))
+              .expectDeletedImages("image1", "image2");
+    }
+
+
+    @Test
+    public void taggedImageWithNoContainersIsUnused() {
+        tester.withExistingImages(image("image-1", null, "vespa-6"))
+              .expectDeletedImages("vespa-6");
+    }
+
+
+    @Test
+    public void unusedImagesWithSimpleImageGc() {
+        tester.withExistingImages(image("parent-image"))
+              .expectDeletedImagesAfterMinutes(30)
+              .withExistingImages(image("parent-image"),
+                                    image("image-1", "parent-image"))
+              .expectDeletedImagesAfterMinutes(0)
+              .expectDeletedImagesAfterMinutes(30)
+              // At this point, parent-image has been unused for 1h, but image-1 depends on parent-image and it has
+              // only been unused for 30m, so we cannot delete parent-image yet. 30 mins later both can be removed
+              .expectDeletedImagesAfterMinutes(30, "image-1", "parent-image");
+    }
+
+
+    @Test
+    public void reDownloadingImageIsNotImmediatelyDeleted() {
+        tester.withExistingImages(image("image"))
+              .expectDeletedImages("image") // After 1h we delete image
+              .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted
+              .expectDeletedImagesAfterMinutes(10)
+              .expectDeletedImages("image"); // 1h after re-download it is deleted again
+    }
+
+
+    @Test
+    public void reDownloadingImageIsNotImmediatelyDeletedWhenDeletingByTag() {
+        tester.withExistingImages(image("image", null, "image-1", "my-tag"))
+              .expectDeletedImages("image-1", "my-tag") // After 1h we delete image
+              .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted
+              .expectDeletedImagesAfterMinutes(10)
+              .expectDeletedImages("image-1", "my-tag"); // 1h after re-download it is deleted again
+    }
+
+    /** Same scenario as in {@link #multipleUnusedImagesAreIdentified()} */
+    @Test
+    public void doesNotDeleteExcludedByIdImages() {
+        tester.withExistingImages(image("parent-image"),
+                                  image("image-1", "parent-image"),
+                                  image("image-2", "parent-image"))
+              // Normally, image-1 and parent-image should also be deleted, but because we exclude image-1
+              // we cannot delete parent-image, so only image-2 is deleted
+              .expectDeletedImages(List.of("image-1"), "image-2");
+    }
+
+    /** Same as in {@link #doesNotDeleteExcludedByIdImages()} but with tags */
+    @Test
+    public void doesNotDeleteExcludedByTagImages() {
+        tester.withExistingImages(image("parent-image", "rhel-6"),
+                                  image("image-1", "parent-image", "vespa:6.288.16"),
+                                  image("image-2", "parent-image", "vespa:6.289.94"))
+              .expectDeletedImages(List.of("vespa:6.288.16"), "vespa:6.289.94");
+    }
+
+    @Test
+    public void exludingNotDownloadedImageIsNoop() {
+        tester.withExistingImages(image("parent-image", "rhel-6"),
+                                  image("image-1", "parent-image", "vespa:6.288.16"),
+                                  image("image-2", "parent-image", "vespa:6.289.94"))
+              .expectDeletedImages(List.of("vespa:6.300.1"), "vespa:6.288.16", "vespa:6.289.94", "rhel-6");
+    }
+
+    private static Image image(String id) {
+        return image(id, null);
+    }
+
+    private static Image image(String id, String parentId, String... tags) {
+        return new Image(id, Optional.ofNullable(parentId), List.of(tags));
+    }
+
+    private static Container container(String name, String imageId) {
+        return new Container(new ContainerId("id-of-" + name), new ContainerName(name),
+                             Container.State.running, imageId, DockerImage.EMPTY, Map.of(),
+                             42, 43, name + ".example.com", ContainerResources.UNLIMITED,
+                             List.of(), true);
+    }
+
+    private static class Tester {
+
+        private final ContainerEngineMock containerEngine = new ContainerEngineMock();
+        private final TaskContext context = new TestTaskContext();
+        private final ManualClock clock = new ManualClock();
+        private final ContainerImagePruner pruner = new ContainerImagePruner(containerEngine, clock);
+        private final Map<String, Integer> removalCountByImageId = new HashMap<>();
+
+        private boolean initialized = false;
+
+        private Tester withExistingImages(Image... images) {
+            containerEngine.setImages(List.of(images));
+            return this;
+        }
+
+        private Tester withExistingContainers(Container... containers) {
+            containerEngine.addContainers(List.of(containers));
+            return this;
+        }
+
+        private Tester expectDeletedImages(String... imageIds) {
+            return expectDeletedImagesAfterMinutes(60, imageIds);
+        }
+
+        private Tester expectDeletedImages(List<String> excludedRefs, String... imageIds) {
+            return expectDeletedImagesAfterMinutes(60, excludedRefs, imageIds);
+        }
+
+        private Tester expectDeletedImagesAfterMinutes(int minutesAfter, String... imageIds) {
+            return expectDeletedImagesAfterMinutes(minutesAfter, Collections.emptyList(), imageIds);
+        }
+
+        private Tester expectDeletedImagesAfterMinutes(int minutesAfter, List<String> excludedRefs, String... imageIds) {
+            if (!initialized) {
+                // Run once with a very long expiry to initialize internal state of existing images
+                pruner.removeUnusedImages(context, List.of(), Duration.ofDays(999));
+                initialized = true;
+            }
+
+            clock.advance(Duration.ofMinutes(minutesAfter));
+
+            pruner.removeUnusedImages(context, excludedRefs, Duration.ofHours(1).minusSeconds(1));
+
+            Arrays.stream(imageIds)
+                  .forEach(imageId -> {
+                      int newValue = removalCountByImageId.getOrDefault(imageId, 0) + 1;
+                      removalCountByImageId.put(imageId, newValue);
+
+                      assertTrue("Image " + imageId + " removed",
+                                 containerEngine.listImages(context).stream().noneMatch(image -> image.id().equals(imageId)));
+                  });
+            return this;
+        }
+    }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerOperationsMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerOperationsMock.java
deleted file mode 100644
index a19e031e41f..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerOperationsMock.java
+++ /dev/null
@@ -1,155 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.integration;
-
-import com.yahoo.config.provision.DockerImage;
-import com.yahoo.vespa.hosted.node.admin.container.Container;
-import com.yahoo.vespa.hosted.node.admin.container.ContainerId;
-import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
-import com.yahoo.vespa.hosted.node.admin.container.ContainerResources;
-import com.yahoo.vespa.hosted.node.admin.container.ContainerStats;
-import com.yahoo.vespa.hosted.node.admin.container.ProcessResult;
-import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials;
-import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
-import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
-import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
-
-import java.time.Duration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
-/**
- * Mock with some simple logic
- *
- * @author freva
- */
-public class ContainerOperationsMock implements ContainerOperations {
-
-    public static final ContainerId CONTAINER_ID = new ContainerId("af345");
-
-    private final Map<ContainerName, Container> containersByContainerName = new HashMap<>();
-    private final Object monitor = new Object();
-
-    @Override
-    public void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources) {
-        synchronized (monitor) {
-            containersByContainerName.put(context.containerName(),
-                                          new Container(CONTAINER_ID,
-                                                        context.hostname().value(),
-                                                        context.node().wantedDockerImage().get(),
-                                                        containerResources,
-                                                        context.containerName(),
-                                                        Container.State.CREATED,
-                                                        2));
-        }
-    }
-
-    @Override
-    public void startContainer(NodeAgentContext context) {
-        synchronized (monitor) {
-            Optional<Container> container = getContainer(context);
-            if (container.isEmpty())  throw new IllegalArgumentException("Cannot start non-existent container " + context.containerName());
-            containersByContainerName.put(context.containerName(), new Container(CONTAINER_ID,
-                                                                                 container.get().hostname,
-                                                                                 container.get().image,
-                                                                                 container.get().resources,
-                                                                                 container.get().name,
-                                                                                 Container.State.RUNNING,
-                                                                                 container.get().pid));
-        }
-    }
-
-    @Override
-    public void removeContainer(NodeAgentContext context, Container container) {
-        synchronized (monitor) {
-            containersByContainerName.remove(container.name);
-        }
-    }
-
-    @Override
-    public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) {
-        synchronized (monitor) {
-            Container container = containersByContainerName.get(context.containerName());
-            containersByContainerName.put(context.containerName(),
-                                          new Container(container.id(), container.hostname, container.image,
-                                                        containerResources, container.name, container.state, container.pid));
-        }
-    }
-
-    @Override
-    public Optional<Container> getContainer(NodeAgentContext context) {
-        synchronized (monitor) {
-            return Optional.ofNullable(containersByContainerName.get(context.containerName()));
-        }
-    }
-
-    @Override
-    public boolean pullImageAsyncIfNeeded(TaskContext context, DockerImage dockerImage, RegistryCredentials registryCredentials) {
-        return false;
-    }
-
-    @Override
-    public ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, String... command) {
-        return null;
-    }
-
-    @Override
-    public ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command) {
-        return null;
-    }
-
-    @Override
-    public CommandResult executeCommandInNetworkNamespace(NodeAgentContext context, String... command) {
-        return null;
-    }
-
-    @Override
-    public String resumeNode(NodeAgentContext context) {
-        return "";
-    }
-
-    @Override
-    public String suspendNode(NodeAgentContext context) {
-        return "";
-    }
-
-    @Override
-    public String restartVespa(NodeAgentContext context) {
-        return "";
-    }
-
-    @Override
-    public String startServices(NodeAgentContext context) {
-        return "";
-    }
-
-    @Override
-    public String stopServices(NodeAgentContext context) {
-        return "";
-    }
-
-    @Override
-    public Optional<ContainerStats> getContainerStats(NodeAgentContext context) {
-        return Optional.empty();
-    }
-
-    @Override
-    public boolean noManagedContainersRunning(TaskContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean retainManagedContainers(TaskContext context, Set<ContainerName> containerNames) {
-        return false;
-    }
-
-    @Override
-    public boolean deleteUnusedContainerImages(TaskContext context, List<DockerImage> excludes, Duration minImageAgeToDelete) {
-        return false;
-    }
-
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java
index 9153afd8e54..4179f53370b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java
@@ -4,11 +4,13 @@ package com.yahoo.vespa.hosted.node.admin.integration;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock;
 import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
 import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials;
 import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
 import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater;
@@ -23,8 +25,6 @@ import org.mockito.InOrder;
 import org.mockito.Mockito;
 
 import java.nio.file.FileSystem;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.time.Clock;
 import java.time.Duration;
 import java.util.Collections;
@@ -46,12 +46,11 @@ public class ContainerTester implements AutoCloseable {
 
     private static final Logger log = Logger.getLogger(ContainerTester.class.getName());
     private static final Duration INTERVAL = Duration.ofMillis(10);
-    private static final Path PATH_TO_VESPA_HOME = Paths.get("/opt/vespa");
     static final HostName HOST_HOSTNAME = HostName.from("host.test.yahoo.com");
 
     private final Thread loopThread;
 
-    final ContainerOperationsMock containerOperations = spy(new ContainerOperationsMock());
+    final ContainerOperations containerOperations = spy(new ContainerOperations(new ContainerEngineMock(), TestFileSystem.create()));
     final NodeRepoMock nodeRepository = spy(new NodeRepoMock());
     final Orchestrator orchestrator = mock(Orchestrator.class);
     final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class);
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
index 3c22305d006..cfb9aaa4001 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
@@ -1,10 +1,10 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 
-import com.yahoo.vespa.hosted.node.admin.container.ProcessResult;
 import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
 import org.junit.Test;
 
 import java.nio.file.Path;
@@ -12,8 +12,8 @@ import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 
-import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL7_DT9;
 import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL7_DT10;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL7_DT9;
 import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8;
 import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.JAVA_HEAP_DUMP_METADATA;
 import static org.junit.Assert.assertEquals;
@@ -91,7 +91,7 @@ public class CoreCollectorTest {
             coreCollector.readBinPathFallback(context, TEST_CORE_PATH);
             fail("Expected not to be able to get bin path");
         } catch (RuntimeException e) {
-            assertEquals("Failed to extract binary path from GDB, result: ProcessResult { exitStatus=1 output= errors=Error 123 }, command: " +
+            assertEquals("Failed to extract binary path from GDB, result: exit status 1, output 'Error 123', command: " +
                     "[/bin/sh, -c, /opt/rh/devtoolset-10/root/bin/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by']", e.getMessage());
         }
     }
@@ -110,7 +110,7 @@ public class CoreCollectorTest {
             coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false);
             fail("Expected not to be able to read backtrace");
         } catch (RuntimeException e) {
-            assertEquals("Failed to read backtrace ProcessResult { exitStatus=1 output= errors=Failure }, Command: " +
+            assertEquals("Failed to read backtrace exit status 1, output 'Failure', Command: " +
                     "[" + GDB_PATH_RHEL7_DT9 + ", -n, -ex, bt, -batch, /usr/bin/program, /tmp/core.1234]", e.getMessage());
         }
     }
@@ -194,6 +194,6 @@ public class CoreCollectorTest {
 
     private void mockExec(NodeAgentContext context, String[] cmd, String output, String error) {
         when(docker.executeCommandInContainerAsRoot(context, cmd))
-                .thenReturn(new ProcessResult(error.isEmpty() ? 0 : 1, output, error));
+                .thenReturn(new CommandResult(null, error.isEmpty() ? 0 : 1, error.isEmpty() ? output : error));
     }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 454374fe3bd..f2b761b614d 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -33,6 +33,7 @@ import org.mockito.InOrder;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
@@ -762,12 +763,17 @@ public class NodeAgentImplTest {
             return dockerImage != null ?
                     Optional.of(new Container(
                             containerId,
-                            hostName,
+                            ContainerName.fromHostname(hostName),
+                            isRunning ? Container.State.running : Container.State.exited,
+                            "image-id-1",
                             dockerImage,
+                            Map.of(),
+                            42,
+                            43,
+                            hostName,
                             containerResources,
-                            ContainerName.fromHostname(hostName),
-                            isRunning ? Container.State.RUNNING : Container.State.EXITED,
-                            isRunning ? 1 : 0)) :
+                            List.of(),
+                            true)) :
                     Optional.empty();
         }).when(containerOperations).getContainer(any());
     }