Add host-life metrics

VESPA-6477

13 files changed, 133 insertions, 51 deletions

diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
index fd4551d0f02..a76c54c65dd 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
@@ -506,6 +506,6 @@ public class DockerImpl implements Docker {
                 .add("host", HostName.getLocalhost())
                 .add("role", "docker").build();
 
-        numberOfDockerDaemonFails = metricReceiver.declareCounter(dimensions, "daemon.api_fails");
+        numberOfDockerDaemonFails = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "daemon.api_fails");
     }
 }
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java
index cb5185e440c..7adcbe715ec 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java
@@ -6,12 +6,12 @@ import java.util.HashMap;
 import java.util.Map;
 
 /**
- * Each metric reported to secret agent has dimmensions.
+ * Each metric reported to secret agent has dimensions.
  *
  * @author valerijf
  */
 public class Dimensions {
-    public final Map<String, Object> dimensionsMap;
+    final Map<String, Object> dimensionsMap;
 
     private Dimensions(Map<String, Object> dimensionsMap) {
         this.dimensionsMap = dimensionsMap;
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java
index 842ad722e5b..776211635fb 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java
@@ -7,21 +7,27 @@ import com.google.inject.Inject;
 import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.metrics.simple.Point;
 
+import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
- * Export metrics to both /state/v1/metrics and makes them available programatically.
+ * Export metrics to both /state/v1/metrics and makes them available programmatically.
+ * Each metric belongs to a yamas application
  *
  * @author valerijf
  */
-public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.DimensionMetrics> {
+public class MetricReceiverWrapper {
+    // Application names used
+    public static final String APPLICATION_DOCKER = "docker";
+    public static final String APPLICATION_HOST_LIFE = "host-life";
+
     private final static ObjectMapper objectMapper = new ObjectMapper();
     private final Object monitor = new Object();
-    private final Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = new HashMap<>();
+    private final Map<String, ApplicationMetrics> applicationMetrics = new HashMap<>(); // key is application name
     private final MetricReceiver metricReceiver;
 
     @Inject
@@ -32,8 +38,9 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim
     /**
      *  Declaring the same dimensions and name results in the same CounterWrapper instance (idempotent).
      */
-    public CounterWrapper declareCounter(Dimensions dimensions, String name) {
+    public CounterWrapper declareCounter(String application, Dimensions dimensions, String name) {
         synchronized (monitor) {
+            Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application);
             if (!metricsByDimensions.containsKey(dimensions)) metricsByDimensions.put(dimensions, new HashMap<>());
             if (!metricsByDimensions.get(dimensions).containsKey(name)) {
                 CounterWrapper counter = new CounterWrapper(metricReceiver.declareCounter(name, new Point(dimensions.dimensionsMap)));
@@ -47,8 +54,9 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim
     /**
      *  Declaring the same dimensions and name results in the same GaugeWrapper instance (idempotent).
      */
-    public GaugeWrapper declareGauge(Dimensions dimensions, String name) {
+    public GaugeWrapper declareGauge(String application, Dimensions dimensions, String name) {
         synchronized (monitor) {
+            Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application);
             if (!metricsByDimensions.containsKey(dimensions))
                 metricsByDimensions.put(dimensions, new HashMap<>());
             if (!metricsByDimensions.get(dimensions).containsKey(name)) {
@@ -62,38 +70,50 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim
 
     public void unsetMetricsForContainer(String hostname) {
         synchronized (monitor) {
-            Iterator<Dimensions> dimensionsIterator = metricsByDimensions.keySet().iterator();
-            while (dimensionsIterator.hasNext()) {
-                Dimensions dimension = dimensionsIterator.next();
-                if (dimension.dimensionsMap.containsKey("host") && dimension.dimensionsMap.get("host").equals(hostname)) {
-                    dimensionsIterator.remove();
-                }
-            }
+            applicationMetrics.values()
+                              .forEach(m -> m.metricsByDimensions.keySet()
+                                                                 .removeIf(d -> d.dimensionsMap.containsKey("host") &&
+                                                                         d.dimensionsMap.get("host").equals(hostname)));
         }
     }
 
+    public List<DimensionMetrics> getMetrics(String application) {
+        synchronized (monitor) {
+            Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application);
+            return metricsByDimensions.entrySet()
+                                     .stream()
+                                     .map(entry -> new DimensionMetrics(application, entry.getKey(), entry.getValue()))
+                                     .collect(Collectors.toList());
+        }
+    }
 
-    @Override
-    public Iterator<DimensionMetrics> iterator() {
+    public List<DimensionMetrics> getAllMetrics() {
         synchronized (monitor) {
-            return metricsByDimensions.entrySet().stream().map(entry ->
-                    new DimensionMetrics(entry.getKey(), entry.getValue())).iterator();
+            List<DimensionMetrics> dimensionMetrics = new ArrayList<>();
+            applicationMetrics.entrySet()
+                              .forEach(e -> e.getValue().metricsByDimensions().entrySet().stream()
+                                             .map(entry -> new DimensionMetrics(e.getKey(), entry.getKey(), entry.getValue()))
+                                             .map(dimensionMetrics::add));
+            return dimensionMetrics;
         }
     }
 
     // For testing
-    Map<String, Number> getMetricsForDimension(Dimensions dimensions) {
+    Map<String, Number> getMetricsForDimension(String application, Dimensions dimensions) {
         synchronized (monitor) {
+            Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application);
             return metricsByDimensions.get(dimensions).entrySet().stream().collect(
                     Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue()));
         }
     }
 
     public class DimensionMetrics {
+        private final String application;
         private final Dimensions dimensions;
         private final Map<String, Object> metrics;
 
-        DimensionMetrics(Dimensions dimensions, Map<String, MetricValue> metricValues) {
+        DimensionMetrics(String application, Dimensions dimensions, Map<String, MetricValue> metricValues) {
+            this.application = application;
             this.dimensions = dimensions;
             this.metrics = metricValues.entrySet().stream().collect(
                     Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue()));
@@ -106,7 +126,7 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim
             routingYamas.put("namespaces", new String[]{"Vespa"});
 
             Map<String, Object> report = new LinkedHashMap<>();
-            report.put("application", "docker");
+            report.put("application", application);
             report.put("timestamp", System.currentTimeMillis() / 1000);
             report.put("dimensions", dimensions.dimensionsMap);
             report.put("metrics", metrics);
@@ -115,4 +135,21 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim
             return objectMapper.writeValueAsString(report);
         }
     }
+
+    private Map<Dimensions, Map<String, MetricValue>> getOrCreateApplicationMetrics(String application) {
+        if (! applicationMetrics.containsKey(application)) {
+            ApplicationMetrics metrics = new ApplicationMetrics();
+            applicationMetrics.put(application, metrics);
+        }
+        return applicationMetrics.get(application).metricsByDimensions();
+    }
+
+    // Application is yamas application, not Vespa application
+    private static class ApplicationMetrics {
+        private final Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = new LinkedHashMap<>();
+
+        Map<Dimensions, Map<String, MetricValue>> metricsByDimensions() {
+            return metricsByDimensions;
+        }
+    }
 }
diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java
index 82de7c8c8d3..1cbc83fb9af 100644
--- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java
+++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java
@@ -14,24 +14,26 @@ import static org.junit.Assert.assertTrue;
  */
 public class MetricReceiverWrapperTest {
     private static final Dimensions hostDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build();
+    private static final String applicationDocker = MetricReceiverWrapper.APPLICATION_DOCKER;
 
     @Test
     public void testDefaultValue() {
         MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
-        metricReceiver.declareCounter(hostDimension, "some.name");
 
-        assertEquals(metricReceiver.getMetricsForDimension(hostDimension).get("some.name"), 0L);
+        metricReceiver.declareCounter(applicationDocker, hostDimension, "some.name");
+
+        assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("some.name"), 0L);
     }
 
     @Test
     public void testSimpleIncrementMetric() {
         MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
-        CounterWrapper counter = metricReceiver.declareCounter(hostDimension, "a_counter.value");
+        CounterWrapper counter = metricReceiver.declareCounter(applicationDocker, hostDimension, "a_counter.value");
 
         counter.add(5);
         counter.add(8);
 
-        Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(hostDimension);
+        Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(applicationDocker, hostDimension);
         assertTrue("Expected only 1 metric value to be set", latestMetrics.size() == 1);
         assertEquals(latestMetrics.get("a_counter.value"), 13L); // 5 + 8
     }
@@ -39,12 +41,12 @@ public class MetricReceiverWrapperTest {
     @Test
     public void testSimpleGauge() {
         MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
-        GaugeWrapper gauge = metricReceiver.declareGauge(hostDimension, "test.gauge");
+        GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge");
 
         gauge.sample(42);
         gauge.sample(-342.23);
 
-        Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(hostDimension);
+        Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(applicationDocker, hostDimension);
         assertTrue("Expected only 1 metric value to be set", latestMetrics.size() == 1);
         assertEquals(latestMetrics.get("test.gauge"), -342.23);
     }
@@ -52,29 +54,29 @@ public class MetricReceiverWrapperTest {
     @Test
     public void testRedeclaringSameGauge() {
         MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
-        GaugeWrapper gauge = metricReceiver.declareGauge(hostDimension, "test.gauge");
+        GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge");
         gauge.sample(42);
 
         // Same as hostDimension, but new instance.
         Dimensions newDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build();
-        GaugeWrapper newGauge = metricReceiver.declareGauge(newDimension, "test.gauge");
+        GaugeWrapper newGauge = metricReceiver.declareGauge(applicationDocker, newDimension, "test.gauge");
         newGauge.sample(56);
 
-        assertEquals(metricReceiver.getMetricsForDimension(hostDimension).get("test.gauge"), 56.);
+        assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("test.gauge"), 56.);
     }
 
     @Test
     public void testSameMetricNameButDifferentDimensions() {
         MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
-        GaugeWrapper gauge = metricReceiver.declareGauge(hostDimension, "test.gauge");
+        GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge");
         gauge.sample(42);
 
         // Not the same as hostDimension.
         Dimensions newDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build();
-        GaugeWrapper newGauge = metricReceiver.declareGauge(newDimension, "test.gauge");
+        GaugeWrapper newGauge = metricReceiver.declareGauge(applicationDocker, newDimension, "test.gauge");
         newGauge.sample(56);
 
-        assertEquals(metricReceiver.getMetricsForDimension(hostDimension).get("test.gauge"), 42.);
-        assertEquals(metricReceiver.getMetricsForDimension(newDimension).get("test.gauge"), 56.);
+        assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("test.gauge"), 42.);
+        assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, newDimension).get("test.gauge"), 56.);
     }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index 5a1edbf0283..d57be59a647 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -351,7 +351,7 @@ public class DockerOperationsImpl implements DockerOperations {
                 .add("host", HostName.getLocalhost())
                 .add("role", "docker").build();
 
-        numberOfRunningContainersGauge = metricReceiver.declareGauge(dimensions, "containers.running");
+        numberOfRunningContainersGauge = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "containers.running");
 
         // Some containers could already be running, count them and initialize to that value
         numberOfRunningContainersGauge.sample(getAllManagedContainers().size());
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 92d33c49f74..2f97f18dcb6 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -59,7 +59,7 @@ public class StorageMaintainer {
                 .add("host", HostName.getLocalhost())
                 .add("role", "docker").build();
 
-        numberOfNodeAdminMaintenanceFails = metricReceiver.declareCounter(dimensions, "nodes.maintenance.fails");
+        numberOfNodeAdminMaintenanceFails = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.maintenance.fails");
     }
 
     public Map<String, Number> updateIfNeededAndGetDiskMetricsFor(ContainerName containerName) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
index ad63f0e8265..0c1402ecfa8 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
@@ -68,9 +68,9 @@ public class NodeAdminImpl implements NodeAdmin {
                 .add("host", HostName.getLocalhost())
                 .add("role", "docker").build();
 
-        this.numberOfContainersInActiveState = metricReceiver.declareGauge(dimensions, "nodes.state.active");
-        this.numberOfContainersInLoadImageState = metricReceiver.declareGauge(dimensions, "nodes.image.loading");
-        this.numberOfUnhandledExceptionsInNodeAgent = metricReceiver.declareCounter(dimensions, "nodes.unhandled_exceptions");
+        this.numberOfContainersInActiveState = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.state.active");
+        this.numberOfContainersInLoadImageState = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.image.loading");
+        this.numberOfUnhandledExceptionsInNodeAgent = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.unhandled_exceptions");
 
         scheduler.scheduleWithFixedDelay(() -> {
             try {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 4c1e400b9bc..bc11162df66 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -505,7 +505,7 @@ public class NodeAgentImpl implements NodeAgent {
         long currentCpuSystemTotalTime = ((Number) stats.getCpuStats().get("system_cpu_usage")).longValue();
 
         double cpuPercentage = lastCpuMetric.getCpuUsagePercentage(currentCpuContainerTotalTime, currentCpuSystemTotalTime);
-        metricReceiver.declareGauge(dimensions, "node.cpu.busy.pct").sample(cpuPercentage);
+        metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "node.cpu.busy.pct").sample(cpuPercentage);
 
         addIfNotNull(dimensions, "node.cpu.throttled_time", stats.getCpuStats().get("throttling_data"), "throttled_time");
         addIfNotNull(dimensions, "node.memory.limit", stats.getMemoryStats(), "limit");
@@ -518,10 +518,14 @@ public class NodeAgentImpl implements NodeAgent {
             addIfNotNull(netDims, "node.network.bytes_sent", interfaceStats, "tx_bytes");
         });
 
+        metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST_LIFE, dimensions, "uptime").sample(lastCpuMetric.getUptime());
+        metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST_LIFE, dimensions, "alive").sample(1);
+
+
         storageMaintainer.ifPresent(maintainer -> maintainer
                 .updateIfNeededAndGetDiskMetricsFor(nodeSpec.containerName)
                 .forEach((metricName, metricValue) ->
-                        metricReceiver.declareGauge(dimensions, metricName).sample(metricValue.doubleValue())));
+                        metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, metricName).sample(metricValue.doubleValue())));
     }
 
     @SuppressWarnings("unchecked")
@@ -529,7 +533,8 @@ public class NodeAgentImpl implements NodeAgent {
         Map<String, Object> metricsMap = (Map<String, Object>) metrics;
         if (metricsMap == null || !metricsMap.containsKey(metricName)) return;
         try {
-            metricReceiver.declareGauge(dimensions, yamasName).sample(((Number) metricsMap.get(metricName)).doubleValue());
+            metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, yamasName)
+                          .sample(((Number) metricsMap.get(metricName)).doubleValue());
         } catch (Throwable e) {
             logger.warning("Failed to update " + yamasName + " metric with value " + metricsMap.get(metricName), e);
         }
@@ -590,6 +595,7 @@ public class NodeAgentImpl implements NodeAgent {
     class CpuUsageReporter {
         private long totalContainerUsage = 0;
         private long totalSystemUsage = 0;
+        private final Instant created = Instant.now();
 
         double getCpuUsagePercentage(long currentContainerUsage, long currentSystemUsage) {
             long deltaSystemUsage = currentSystemUsage - totalSystemUsage;
@@ -600,6 +606,10 @@ public class NodeAgentImpl implements NodeAgent {
             totalSystemUsage = currentSystemUsage;
             return cpuUsagePct;
         }
+
+        double getUptime() {
+            return Duration.between(created, Instant.now()).toMillis() * 1000;
+        }
     }
 
     // TODO: Also skip orchestration if we're downgrading in test/staging
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
index 45367b74c29..c81442e5a54 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
@@ -46,7 +46,7 @@ public class ComponentsProviderImpl implements ComponentsProvider {
     private static final int NODE_AGENT_SCAN_INTERVAL_MILLIS = 30000;
     private static final int WEB_SERVICE_PORT = Defaults.getDefaults().vespaWebServicePort();
     // We only scan for new nodes within a host every 5 minutes. This is only if new nodes are added or removed
-    // which happens rarely. Changes of apps running etc it detected by the NodeAgent.
+    // which happens rarely. Changes of apps running etc. is detected by the NodeAgent.
     private static final int NODE_ADMIN_STATE_INTERVAL_MILLIS = 5 * 60000;
 
     public ComponentsProviderImpl(Docker docker, MetricReceiverWrapper metricReceiver, Environment environment,
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java
index 0aa0c2f7c0b..f273b1b1521 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java
@@ -68,7 +68,7 @@ public class RestApiHandler extends LoggingRequestHandler{
                 @Override
                 public void render(OutputStream outputStream) throws IOException {
                     try (PrintStream printStream = new PrintStream(outputStream)) {
-                        for (MetricReceiverWrapper.DimensionMetrics dimensionMetrics : metricReceiverWrapper) {
+                        for (MetricReceiverWrapper.DimensionMetrics dimensionMetrics : metricReceiverWrapper.getAllMetrics()) {
                             String secretAgentJsonReport = dimensionMetrics.toSecretAgentReport() + "\n";
                             printStream.write(secretAgentJsonReport.getBytes(StandardCharsets.UTF_8.name()));
                         }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMaker.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMaker.java
index 950d45fc6fe..6034df2d82c 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMaker.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMaker.java
@@ -15,15 +15,15 @@ import java.util.Map;
 public class SecretAgentScheduleMaker {
     private final String id;
     private final int interval;
-    private final Path checkExecuteable;
+    private final Path checkExecutable;
     private final String[] arguments;
     private String user = "nobody";
     private final Map<String, Object> tags = new LinkedHashMap<>();
 
-    public SecretAgentScheduleMaker(String id, int interval, Path checkExecuteable, String... arguments) {
+    public SecretAgentScheduleMaker(String id, int interval, Path checkExecutable, String... arguments) {
         this.id = id;
         this.interval = interval;
-        this.checkExecuteable = checkExecuteable;
+        this.checkExecutable = checkExecutable;
         this.arguments = arguments;
     }
 
@@ -49,7 +49,7 @@ public class SecretAgentScheduleMaker {
                 .append("- id: ").append(id).append("\n")
                 .append("  interval: ").append(interval).append("\n")
                 .append("  user: ").append(user).append("\n")
-                .append("  check: ").append(checkExecuteable.toFile()).append("\n");
+                .append("  check: ").append(checkExecutable.toFile()).append("\n");
 
         if (arguments.length > 0) {
             stringBuilder.append("  args: \n");
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 983e4764fa8..447097fe337 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -520,12 +520,19 @@ public class NodeAgentImplTest {
         nodeAgent.updateContainerNodeMetrics();
 
         Set<Map<String, Object>> actualMetrics = new HashSet<>();
-        for (MetricReceiverWrapper.DimensionMetrics dimensionMetrics : metricReceiver) {
+        for (MetricReceiverWrapper.DimensionMetrics dimensionMetrics : metricReceiver.getMetrics(MetricReceiverWrapper.APPLICATION_DOCKER)) {
             Map<String, Object> metrics = objectMapper.readValue(dimensionMetrics.toSecretAgentReport(), Map.class);
             metrics.remove("timestamp"); // Remove timestamp so we can test against expected map
             actualMetrics.add(metrics);
         }
 
+        for (MetricReceiverWrapper.DimensionMetrics dimensionMetrics : metricReceiver.getMetrics(MetricReceiverWrapper.APPLICATION_HOST_LIFE)) {
+            Map<String, Object> metrics = objectMapper.readValue(dimensionMetrics.toSecretAgentReport(), Map.class);
+            metrics.remove("timestamp"); // Remove timestamp so we can test against expected map
+            ((Map)metrics.get("metrics")).remove("uptime"); // Remove uptime so we can test against expected map. Implicit test of field existing too
+            actualMetrics.add(metrics);
+        }
+
         File expectedMetricsFile = new File(classLoader.getResource("docker.stats.metrics.expected.json").getFile());
         Set<Map<String, Object>> expectedMetrics = objectMapper.readValue(expectedMetricsFile, Set.class);
 
diff --git a/node-admin/src/test/resources/docker.stats.metrics.expected.json b/node-admin/src/test/resources/docker.stats.metrics.expected.json
index 50a47a9e14a..ac6761aec23 100644
--- a/node-admin/src/test/resources/docker.stats.metrics.expected.json
+++ b/node-admin/src/test/resources/docker.stats.metrics.expected.json
@@ -1,5 +1,31 @@
 [
   {
+    "application": "host-life",
+    "dimensions": {
+      "flavor": "docker",
+      "app": "testapp.testinstance",
+      "clustertype": "clustType",
+      "role": "tenants",
+      "tenantName": "tester",
+      "zone": "dev.us-east-1",
+      "host": "hostname",
+      "vespaVersion": "1.2.3",
+      "state": "active",
+      "clusterid": "clustId",
+      "parentHostname": "parent.host.name.yahoo.com"
+    },
+    "metrics": {
+      "alive": 1.0
+    },
+    "routing": {
+      "yamas": {
+        "namespaces": [
+          "Vespa"
+        ]
+      }
+    }
+  },
+  {
     "application":"docker",
     "dimensions":{
       "flavor":"docker",