Move updateContainerNodeMetrics() to internal code

5 files changed, 9 insertions, 785 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 90eda96d445..e9b48e89fd9 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.nodeagent;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.zone.ZoneApi;
@@ -15,10 +14,6 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
 import com.yahoo.vespa.hosted.dockerapi.ContainerStats;
 import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException;
 import com.yahoo.vespa.hosted.dockerapi.exception.DockerException;
-import com.yahoo.vespa.hosted.dockerapi.exception.DockerExecTimeoutException;
-import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics;
-import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions;
-import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
@@ -31,11 +26,8 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer;
 import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
-import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig;
 
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -95,7 +87,6 @@ public class NodeAgentImpl implements NodeAgent {
     private ContainerState containerState = UNKNOWN;
 
     private NodeSpec lastNode = null;
-    private CpuUsageReporter lastCpuMetric = new CpuUsageReporter();
 
     // Created in NodeAdminImpl
     public NodeAgentImpl(
@@ -209,7 +200,6 @@ public class NodeAgentImpl implements NodeAgent {
         ContainerData containerData = createContainerData(context);
         dockerOperations.createContainer(context, containerData, getContainerResources(context));
         dockerOperations.startContainer(context);
-        lastCpuMetric = new CpuUsageReporter();
 
         hasStartedServices = true; // Automatically started with the container
         hasResumedNode = false;
@@ -513,98 +503,12 @@ public class NodeAgentImpl implements NodeAgent {
         }
     }
 
-    @SuppressWarnings("unchecked")
+    @Override
     public void updateContainerNodeMetrics() {
-        if (containerState != UNKNOWN) return;
-        final NodeAgentContext context = contextSupplier.currentContext();
-        final NodeSpec node = context.node();
-
-        Optional<ContainerStats> containerStats = dockerOperations.getContainerStats(context);
-        if (!containerStats.isPresent()) return;
-
-        Dimensions.Builder dimensionsBuilder = new Dimensions.Builder()
-                .add("host", context.hostname().value())
-                .add("role", SecretAgentCheckConfig.nodeTypeToRole(context.nodeType()))
-                .add("state", node.state().toString());
-        node.parentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent));
-        node.allowedToBeDown().ifPresent(allowed ->
-                dimensionsBuilder.add("orchestratorState", allowed ? "ALLOWED_TO_BE_DOWN" : "NO_REMARKS"));
-        Dimensions dimensions = dimensionsBuilder.build();
-
-        ContainerStats stats = containerStats.get();
-        final String APP = Metrics.APPLICATION_NODE;
-        final int totalNumCpuCores = stats.getCpuStats().getOnlineCpus();
-        final long memoryTotalBytes = stats.getMemoryStats().getLimit();
-        final long memoryTotalBytesUsage = stats.getMemoryStats().getUsage();
-        final long memoryTotalBytesCache = stats.getMemoryStats().getCache();
-        final long diskTotalBytes = (long) (node.diskGb() * BYTES_IN_GB);
-        final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(context);
-
-        lastCpuMetric.updateCpuDeltas(stats.getCpuStats());
-
-        // Ratio of CPU cores allocated to this container to total number of CPU cores on this host
-        final double allocatedCpuRatio = node.vcpus() / totalNumCpuCores;
-        double cpuUsageRatioOfAllocated = lastCpuMetric.getCpuUsageRatio() / allocatedCpuRatio;
-        double cpuKernelUsageRatioOfAllocated = lastCpuMetric.getCpuKernelUsageRatio() / allocatedCpuRatio;
-        double cpuThrottledTimeRate = lastCpuMetric.getThrottledTimeRate();
-        double cpuThrottledCpuTimeRate = lastCpuMetric.getThrottledCpuTimeRate();
-
-        long memoryTotalBytesUsed = memoryTotalBytesUsage - memoryTotalBytesCache;
-        double memoryUsageRatio = (double) memoryTotalBytesUsed / memoryTotalBytes;
-        double memoryTotalUsageRatio = (double) memoryTotalBytesUsage / memoryTotalBytes;
-        Optional<Double> diskUsageRatio = diskTotalBytesUsed.map(used -> (double) used / diskTotalBytes);
-
-        List<DimensionMetrics> metrics = new ArrayList<>();
-        DimensionMetrics.Builder systemMetricsBuilder = new DimensionMetrics.Builder(APP, dimensions)
-                .withMetric("mem.limit", memoryTotalBytes)
-                .withMetric("mem.used", memoryTotalBytesUsed)
-                .withMetric("mem.util", 100 * memoryUsageRatio)
-                .withMetric("mem_total.used", memoryTotalBytesUsage)
-                .withMetric("mem_total.util", 100 * memoryTotalUsageRatio)
-                .withMetric("cpu.util", 100 * cpuUsageRatioOfAllocated)
-                .withMetric("cpu.sys.util", 100 * cpuKernelUsageRatioOfAllocated)
-                .withMetric("cpu.throttled_time.rate", cpuThrottledTimeRate)
-                .withMetric("cpu.throttled_cpu_time.rate", cpuThrottledCpuTimeRate)
-                .withMetric("cpu.vcpus", node.vcpus())
-                .withMetric("disk.limit", diskTotalBytes);
-
-        diskTotalBytesUsed.ifPresent(diskUsed -> systemMetricsBuilder.withMetric("disk.used", diskUsed));
-        diskUsageRatio.ifPresent(diskRatio -> systemMetricsBuilder.withMetric("disk.util", 100 * diskRatio));
-        metrics.add(systemMetricsBuilder.build());
-
-        stats.getNetworks().forEach((interfaceName, interfaceStats) -> {
-            Dimensions netDims = dimensionsBuilder.add("interface", interfaceName).build();
-            DimensionMetrics networkMetrics = new DimensionMetrics.Builder(APP, netDims)
-                    .withMetric("net.in.bytes", interfaceStats.getRxBytes())
-                    .withMetric("net.in.errors", interfaceStats.getRxErrors())
-                    .withMetric("net.in.dropped", interfaceStats.getRxDropped())
-                    .withMetric("net.out.bytes", interfaceStats.getTxBytes())
-                    .withMetric("net.out.errors", interfaceStats.getTxErrors())
-                    .withMetric("net.out.dropped", interfaceStats.getTxDropped())
-                    .build();
-            metrics.add(networkMetrics);
-        });
-
-        pushMetricsToContainer(context, metrics);
-    }
-
-    private void pushMetricsToContainer(NodeAgentContext context, List<DimensionMetrics> metrics) {
-        StringBuilder params = new StringBuilder();
-        try {
-            for (DimensionMetrics dimensionMetrics : metrics) {
-                params.append(dimensionMetrics.toSecretAgentReport());
-            }
-            String wrappedMetrics = "s:" + params.toString();
-
-            // Push metrics to the metrics proxy in each container.
-            // TODO Remove port selection logic when all hosted apps have upgraded to Vespa 7.
-            int port = context.node().currentVespaVersion().map(version -> version.getMajor() == 6).orElse(false) ? 19091 : 19095;
-            String[] command = {"vespa-rpc-invoke",  "-t", "2",  "tcp/localhost:" + port,  "setExtraMetrics", wrappedMetrics};
-            dockerOperations.executeCommandInContainerAsRoot(context, 5L, command);
-        } catch (JsonProcessingException | DockerExecTimeoutException  e) {
-            context.log(logger, LogLevel.WARNING, "Failed to push metrics to container", e);
-        }
-
+        NodeAgentContext context = contextSupplier.currentContext();
+        Optional<ContainerStats> containerStats = containerState != UNKNOWN ?
+                dockerOperations.getContainerStats(context) : Optional.empty();
+        updateContainerNodeMetrics(context, containerStats);
     }
 
     private Optional<Container> getContainer(NodeAgentContext context) {
@@ -621,66 +525,6 @@ public class NodeAgentImpl implements NodeAgent {
         return temp;
     }
 
-    class CpuUsageReporter {
-        private static final double PERIOD_IN_NANOSECONDS = 1_000d * ContainerResources.CPU_PERIOD_US;
-        private long containerKernelUsage = 0;
-        private long totalContainerUsage = 0;
-        private long totalSystemUsage = 0;
-        private long throttledTime = 0;
-        private long throttlingActivePeriods = 0;
-        private long throttledPeriods = 0;
-
-        private long deltaContainerKernelUsage;
-        private long deltaContainerUsage;
-        private long deltaSystemUsage;
-        private long deltaThrottledTime;
-        private long deltaThrottlingActivePeriods;
-        private long deltaThrottledPeriods;
-
-        private void updateCpuDeltas(ContainerStats.CpuStats cpuStats) {
-            // Do not calculate delta during the first tick - that will result in a metric value that is
-            // average since container start
-            if (totalSystemUsage != 0) {
-                deltaSystemUsage = cpuStats.getSystemCpuUsage() - totalSystemUsage;
-                deltaContainerUsage = cpuStats.getTotalUsage() - totalContainerUsage;
-                deltaContainerKernelUsage = cpuStats.getUsageInKernelMode() - containerKernelUsage;
-                deltaThrottledTime = cpuStats.getThrottledTime() - throttledTime;
-                deltaThrottlingActivePeriods = cpuStats.getThrottlingActivePeriods() - throttlingActivePeriods;
-                deltaThrottledPeriods = cpuStats.getThrottledPeriods() - throttledPeriods;
-            }
-
-            totalSystemUsage = cpuStats.getSystemCpuUsage();
-            totalContainerUsage = cpuStats.getTotalUsage();
-            containerKernelUsage = cpuStats.getUsageInKernelMode();
-            throttledTime = cpuStats.getThrottledTime();
-            throttlingActivePeriods = cpuStats.getThrottlingActivePeriods();
-            throttledPeriods = cpuStats.getThrottledPeriods();
-        }
-
-        /**
-         * Returns the CPU usage ratio for the docker container that this NodeAgent is managing
-         * in the time between the last two times updateCpuDeltas() was called. This is calculated
-         * by dividing the CPU time used by the container with the CPU time used by the entire system.
-         */
-        double getCpuUsageRatio() {
-            return deltaSystemUsage == 0 ? Double.NaN : (double) deltaContainerUsage / deltaSystemUsage;
-        }
-
-        double getCpuKernelUsageRatio() {
-            return deltaSystemUsage == 0 ? Double.NaN : (double) deltaContainerKernelUsage / deltaSystemUsage;
-        }
-
-        double getThrottledTimeRate() {
-            return deltaThrottlingActivePeriods == 0 ? Double.NaN :
-                    (double) deltaThrottledPeriods / deltaThrottlingActivePeriods;
-        }
-
-        double getThrottledCpuTimeRate() {
-            return deltaThrottlingActivePeriods == 0 ? Double.NaN :
-                    deltaThrottledTime / (PERIOD_IN_NANOSECONDS * deltaThrottlingActivePeriods);
-        }
-    }
-
     // TODO: Also skip orchestration if we're downgrading in test/staging
     // How to implement:
     //  - test/staging: We need to figure out whether we're in test/staging, zone is available in Environment
@@ -726,4 +570,8 @@ public class NodeAgentImpl implements NodeAgent {
             }
         };
     }
+
+    protected void updateContainerNodeMetrics(NodeAgentContext context, Optional<ContainerStats> containerStats) {
+
+    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 46af7e7bafd..f26d0b57d72 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -9,12 +9,8 @@ import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.dockerapi.Container;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
-import com.yahoo.vespa.hosted.dockerapi.ContainerStats;
 import com.yahoo.vespa.hosted.dockerapi.exception.DockerException;
-import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
@@ -28,16 +24,8 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
 import org.junit.Test;
 import org.mockito.InOrder;
 
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
 import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
-import static com.yahoo.yolean.Exceptions.uncheck;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
@@ -76,7 +64,6 @@ public class NodeAgentImplTest {
     private final NodeRepository nodeRepository = mock(NodeRepository.class);
     private final Orchestrator orchestrator = mock(Orchestrator.class);
     private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class);
-    private final Metrics metrics = new Metrics();
     private final AclMaintainer aclMaintainer = mock(AclMaintainer.class);
     private final HealthChecker healthChecker = mock(HealthChecker.class);
     private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class);
@@ -638,81 +625,6 @@ public class NodeAgentImplTest {
     }
 
     @Test
-    @SuppressWarnings("unchecked")
-    public void testGetRelevantMetrics() throws Exception {
-        String json = Files.readString(Paths.get("src/test/resources/docker.stats.json"));
-        ContainerStats stats2 = ContainerStats.fromJson(json);
-        ContainerStats stats1 = ContainerStats.fromJson(json.replace("\"cpu_stats\"", "\"cpu_stats2\"").replace("\"precpu_stats\"", "\"cpu_stats\""));
-
-        NodeOwner owner = new NodeOwner("tester", "testapp", "testinstance");
-        NodeMembership membership = new NodeMembership("clustType", "clustId", "grp", 3, false);
-        final NodeSpec node = nodeBuilder
-                .wantedDockerImage(dockerImage)
-                .currentDockerImage(dockerImage)
-                .state(NodeState.active)
-                .currentVespaVersion(vespaVersion)
-                .owner(owner)
-                .membership(membership)
-                .memoryGb(2)
-                .allowedToBeDown(true)
-                .parentHostname("parent.host.name.yahoo.com")
-                .build();
-
-        NodeAgentContext context = createContext(node);
-        NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
-
-        when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node));
-        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(39625000000L));
-        when(dockerOperations.getContainerStats(eq(context)))
-                .thenReturn(Optional.of(stats1))
-                .thenReturn(Optional.of(stats2));
-
-        List<String> expectedMetrics = Stream.of(0, 1)
-                .map(i -> Paths.get("src/test/resources/expected.container.system.metrics." + i + ".txt"))
-                .map(path -> uncheck(() -> Files.readString(path)))
-                .map(content -> content.replaceAll("\\s", "").replaceAll("\\n", ""))
-                .collect(Collectors.toList());
-        int[] counter = {0};
-
-        doAnswer(invocation -> {
-            NodeAgentContext calledContainerName = (NodeAgentContext) invocation.getArguments()[0];
-            long calledTimeout = (long) invocation.getArguments()[1];
-            String[] calledCommand = new String[invocation.getArguments().length - 2];
-            System.arraycopy(invocation.getArguments(), 2, calledCommand, 0, calledCommand.length);
-            calledCommand[calledCommand.length - 1] = calledCommand[calledCommand.length - 1]
-                    .replaceAll("\"timestamp\":\\d+", "\"timestamp\":0")
-                    .replaceAll("([0-9]+\\.[0-9]{1,3})([0-9]*)", "$1"); // Only keep the first 3 decimals
-
-            assertEquals(context, calledContainerName);
-            assertEquals(5L, calledTimeout);
-            String[] expectedCommand = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:19095",
-                    "setExtraMetrics", expectedMetrics.get(counter[0])};
-            assertArrayEquals("Ivocation #" + counter[0], expectedCommand, calledCommand);
-            counter[0]++;
-            return null;
-        }).when(dockerOperations).executeCommandInContainerAsRoot(any(), any(), any());
-
-        nodeAgent.updateContainerNodeMetrics();
-        nodeAgent.updateContainerNodeMetrics();
-    }
-
-    @Test
-    public void testGetRelevantMetricsForReadyNode() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.ready)
-                .build();
-
-        NodeAgentContext context = createContext(node);
-        NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
-
-        when(dockerOperations.getContainerStats(eq(context))).thenReturn(Optional.empty());
-
-        nodeAgent.updateContainerNodeMetrics();
-
-        assertEquals(List.of(), metrics.getDefaultMetrics());
-    }
-
-    @Test
     public void testRunningConfigServer() {
         final NodeSpec node = nodeBuilder
                 .type(NodeType.config)
diff --git a/node-admin/src/test/resources/docker.stats.json b/node-admin/src/test/resources/docker.stats.json
deleted file mode 100644
index 5b42d9a2428..00000000000
--- a/node-admin/src/test/resources/docker.stats.json
+++ /dev/null
@@ -1,376 +0,0 @@
-{
-  "read":"2016-10-05T07:28:17.228361751Z",
-  "precpu_stats":{
-    "cpu_usage":{
-      "total_usage":332026268600,
-      "percpu_usage":[
-        46767331190,
-        46637593621,
-        36196010351,
-        38846420953,
-        44237804850,
-        35751912062,
-        44546685143,
-        39042510430
-      ],
-      "usage_in_kernelmode":44040000000,
-      "usage_in_usermode":158940000000
-    },
-    "system_cpu_usage":5876874910000000,
-    "throttling_data":{
-      "periods":820694,
-      "throttled_periods":177731,
-      "throttled_time":81891944744550
-    }
-  },
-  "cpu_stats":{
-    "cpu_usage":{
-      "total_usage":332131163600,
-      "percpu_usage":[
-        46774042376,
-        46639549207,
-        36204341756,
-        38879138416,
-        44256253747,
-        35760081676,
-        44567860460,
-        39049895962
-      ],
-      "usage_in_kernelmode":44106083850,
-      "usage_in_usermode":158950000000
-    },
-    "system_cpu_usage":5876882680000000,
-    "throttling_data":{
-      "periods":821264,
-      "throttled_periods":178201,
-      "throttled_time":82181944744550
-    }
-  },
-  "memory_stats":{
-    "usage":1752707072,
-    "max_usage":1818116096,
-    "stats":{
-      "active_anon":1326051328,
-      "active_file":188919808,
-      "cache":678965248,
-      "hierarchical_memory_limit":4294967296,
-      "hierarchical_memsw_limit":8589934592,
-      "inactive_anon":0,
-      "inactive_file":237735936,
-      "mapped_file":62976000,
-      "pgfault":3102812,
-      "pgmajfault":1403,
-      "pgpgin":1691151,
-      "pgpgout":1263244,
-      "rss":1326026752,
-      "rss_huge":0,
-      "swap":0,
-      "total_active_anon":1326051328,
-      "total_active_file":188919808,
-      "total_cache":426680320,
-      "total_inactive_anon":0,
-      "total_inactive_file":237735936,
-      "total_mapped_file":62976000,
-      "total_pgfault":3102812,
-      "total_pgmajfault":1403,
-      "total_pgpgin":1691151,
-      "total_pgpgout":1263244,
-      "total_rss":1326026752,
-      "total_rss_huge":0,
-      "total_swap":0,
-      "total_unevictable":0,
-      "unevictable":0
-    },
-    "failcnt":0,
-    "limit":4294967296
-  },
-  "blkio_stats":{
-    "io_service_bytes_recursive":[
-      {
-        "major":252,
-        "minor":0,
-        "op":"Read",
-        "value":53248
-      },
-      {
-        "major":252,
-        "minor":0,
-        "op":"Write",
-        "value":602112
-      },
-      {
-        "major":252,
-        "minor":0,
-        "op":"Sync",
-        "value":0
-      },
-      {
-        "major":252,
-        "minor":0,
-        "op":"Async",
-        "value":655360
-      },
-      {
-        "major":252,
-        "minor":0,
-        "op":"Total",
-        "value":655360
-      },
-      {
-        "major":7,
-        "minor":0,
-        "op":"Read",
-        "value":308224
-      },
-      {
-        "major":7,
-        "minor":0,
-        "op":"Write",
-        "value":573440
-      },
-      {
-        "major":7,
-        "minor":0,
-        "op":"Sync",
-        "value":0
-      },
-      {
-        "major":7,
-        "minor":0,
-        "op":"Async",
-        "value":881664
-      },
-      {
-        "major":7,
-        "minor":0,
-        "op":"Total",
-        "value":881664
-      },
-      {
-        "major":253,
-        "minor":0,
-        "op":"Read",
-        "value":308224
-      },
-      {
-        "major":253,
-        "minor":0,
-        "op":"Write",
-        "value":573440
-      },
-      {
-        "major":253,
-        "minor":0,
-        "op":"Sync",
-        "value":0
-      },
-      {
-        "major":253,
-        "minor":0,
-        "op":"Async",
-        "value":881664
-      },
-      {
-        "major":253,
-        "minor":0,
-        "op":"Total",
-        "value":881664
-      },
-      {
-        "major":253,
-        "minor":3,
-        "op":"Read",
-        "value":343847936
-      },
-      {
-        "major":253,
-        "minor":3,
-        "op":"Write",
-        "value":786432
-      },
-      {
-        "major":253,
-        "minor":3,
-        "op":"Sync",
-        "value":131072
-      },
-      {
-        "major":253,
-        "minor":3,
-        "op":"Async",
-        "value":344503296
-      },
-      {
-        "major":253,
-        "minor":3,
-        "op":"Total",
-        "value":344634368
-      }
-    ],
-    "io_serviced_recursive":[
-      {
-        "major":252,
-        "minor":0,
-        "op":"Read",
-        "value":13
-      },
-      {
-        "major":252,
-        "minor":0,
-        "op":"Write",
-        "value":147
-      },
-      {
-        "major":252,
-        "minor":0,
-        "op":"Sync",
-        "value":0
-      },
-      {
-        "major":252,
-        "minor":0,
-        "op":"Async",
-        "value":160
-      },
-      {
-        "major":252,
-        "minor":0,
-        "op":"Total",
-        "value":160
-      },
-      {
-        "major":7,
-        "minor":0,
-        "op":"Read",
-        "value":37
-      },
-      {
-        "major":7,
-        "minor":0,
-        "op":"Write",
-        "value":124
-      },
-      {
-        "major":7,
-        "minor":0,
-        "op":"Sync",
-        "value":0
-      },
-      {
-        "major":7,
-        "minor":0,
-        "op":"Async",
-        "value":161
-      },
-      {
-        "major":7,
-        "minor":0,
-        "op":"Total",
-        "value":161
-      },
-      {
-        "major":253,
-        "minor":0,
-        "op":"Read",
-        "value":37
-      },
-      {
-        "major":253,
-        "minor":0,
-        "op":"Write",
-        "value":124
-      },
-      {
-        "major":253,
-        "minor":0,
-        "op":"Sync",
-        "value":0
-      },
-      {
-        "major":253,
-        "minor":0,
-        "op":"Async",
-        "value":161
-      },
-      {
-        "major":253,
-        "minor":0,
-        "op":"Total",
-        "value":161
-      },
-      {
-        "major":253,
-        "minor":3,
-        "op":"Read",
-        "value":11812
-      },
-      {
-        "major":253,
-        "minor":3,
-        "op":"Write",
-        "value":142
-      },
-      {
-        "major":253,
-        "minor":3,
-        "op":"Sync",
-        "value":2
-      },
-      {
-        "major":253,
-        "minor":3,
-        "op":"Async",
-        "value":11952
-      },
-      {
-        "major":253,
-        "minor":3,
-        "op":"Total",
-        "value":11954
-      }
-    ],
-    "io_queue_recursive":[
-
-    ],
-    "io_service_time_recursive":[
-
-    ],
-    "io_wait_time_recursive":[
-
-    ],
-    "io_merged_recursive":[
-
-    ],
-    "io_time_recursive":[
-
-    ],
-    "sectors_recursive":[
-
-    ]
-  },
-  "pids_stats":{
-
-  },
-  "networks":{
-    "eth0":{
-      "rx_bytes":19499270,
-      "rx_packets":58913,
-      "rx_errors":55,
-      "rx_dropped":4,
-      "tx_bytes":20303455,
-      "tx_packets":62319,
-      "tx_errors":3,
-      "tx_dropped":13
-    },
-    "eth1":{
-      "rx_bytes":3245766,
-      "rx_packets":23462,
-      "rx_errors":0,
-      "rx_dropped":0,
-      "tx_bytes":54246745,
-      "tx_packets":34562,
-      "tx_errors":0,
-      "tx_dropped":0
-    }
-  }
-}
\ No newline at end of file
diff --git a/node-admin/src/test/resources/expected.container.system.metrics.0.txt b/node-admin/src/test/resources/expected.container.system.metrics.0.txt
deleted file mode 100644
index ea6036ce2ea..00000000000
--- a/node-admin/src/test/resources/expected.container.system.metrics.0.txt
+++ /dev/null
@@ -1,78 +0,0 @@
-s:
-{
-    "application": "vespa.node",
-    "dimensions": {
-        "host": "host1.test.yahoo.com",
-        "orchestratorState":"ALLOWED_TO_BE_DOWN",
-        "parentHostname": "parent.host.name.yahoo.com",
-        "role": "tenants",
-        "state": "active"
-    },
-    "metrics": {
-        "cpu.vcpus": 2.0,
-        "disk.limit": 250000000000,
-        "disk.used": 39625000000,
-        "disk.util": 15.85,
-        "mem.limit": 4294967296,
-        "mem.used": 1073741824,
-        "mem.util": 25.0,
-        "mem_total.used": 1752707072,
-        "mem_total.util": 40.808
-    },
-    "routing": {
-        "yamas": {
-            "namespaces": ["Vespa"]
-        }
-    },
-    "timestamp": 0
-}
-{
-    "application": "vespa.node",
-    "dimensions": {
-        "host": "host1.test.yahoo.com",
-        "interface": "eth0",
-        "orchestratorState":"ALLOWED_TO_BE_DOWN",
-        "parentHostname": "parent.host.name.yahoo.com",
-        "role": "tenants",
-        "state": "active"
-    },
-    "metrics": {
-        "net.in.bytes": 19499270,
-        "net.in.dropped": 4,
-        "net.in.errors": 55,
-        "net.out.bytes": 20303455,
-        "net.out.dropped": 13,
-        "net.out.errors": 3
-    },
-    "routing": {
-        "yamas": {
-            "namespaces": ["Vespa"]
-        }
-    },
-    "timestamp": 0
-}
-{
-    "application": "vespa.node",
-    "dimensions": {
-        "host": "host1.test.yahoo.com",
-        "interface": "eth1",
-        "orchestratorState":"ALLOWED_TO_BE_DOWN",
-        "parentHostname": "parent.host.name.yahoo.com",
-        "role": "tenants",
-        "state": "active"
-    },
-    "metrics": {
-        "net.in.bytes": 3245766,
-        "net.in.dropped": 0,
-        "net.in.errors": 0,
-        "net.out.bytes": 54246745,
-        "net.out.dropped": 0,
-        "net.out.errors": 0
-    },
-    "routing": {
-        "yamas": {
-            "namespaces": ["Vespa"]
-        }
-    },
-    "timestamp": 0
-}
\ No newline at end of file
diff --git a/node-admin/src/test/resources/expected.container.system.metrics.1.txt b/node-admin/src/test/resources/expected.container.system.metrics.1.txt
deleted file mode 100644
index 54d4d36c7d0..00000000000
--- a/node-admin/src/test/resources/expected.container.system.metrics.1.txt
+++ /dev/null
@@ -1,82 +0,0 @@
-s:
-{
-    "application": "vespa.node",
-    "dimensions": {
-        "host": "host1.test.yahoo.com",
-        "orchestratorState":"ALLOWED_TO_BE_DOWN",
-        "parentHostname": "parent.host.name.yahoo.com",
-        "role": "tenants",
-        "state": "active"
-    },
-    "metrics": {
-        "cpu.sys.util": 3.402,
-        "cpu.throttled_cpu_time.rate": 5.087,
-        "cpu.throttled_time.rate": 0.824,
-        "cpu.util": 5.4,
-        "cpu.vcpus": 2.0,
-        "disk.limit": 250000000000,
-        "disk.used": 39625000000,
-        "disk.util": 15.85,
-        "mem.limit": 4294967296,
-        "mem.used": 1073741824,
-        "mem.util": 25.0,
-        "mem_total.used": 1752707072,
-        "mem_total.util": 40.808
-    },
-    "routing": {
-        "yamas": {
-            "namespaces": ["Vespa"]
-        }
-    },
-    "timestamp": 0
-}
-{
-    "application": "vespa.node",
-    "dimensions": {
-        "host": "host1.test.yahoo.com",
-        "interface": "eth0",
-        "orchestratorState":"ALLOWED_TO_BE_DOWN",
-        "parentHostname": "parent.host.name.yahoo.com",
-        "role": "tenants",
-        "state": "active"
-    },
-    "metrics": {
-        "net.in.bytes": 19499270,
-        "net.in.dropped": 4,
-        "net.in.errors": 55,
-        "net.out.bytes": 20303455,
-        "net.out.dropped": 13,
-        "net.out.errors": 3
-    },
-    "routing": {
-        "yamas": {
-            "namespaces": ["Vespa"]
-        }
-    },
-    "timestamp": 0
-}
-{
-    "application": "vespa.node",
-    "dimensions": {
-        "host": "host1.test.yahoo.com",
-        "interface": "eth1",
-        "orchestratorState":"ALLOWED_TO_BE_DOWN",
-        "parentHostname": "parent.host.name.yahoo.com",
-        "role": "tenants",
-        "state": "active"
-    },
-    "metrics": {
-        "net.in.bytes": 3245766,
-        "net.in.dropped": 0,
-        "net.in.errors": 0,
-        "net.out.bytes": 54246745,
-        "net.out.dropped": 0,
-        "net.out.errors": 0
-    },
-    "routing": {
-        "yamas": {
-            "namespaces": ["Vespa"]
-        }
-    },
-    "timestamp": 0
-}
\ No newline at end of file