Add docker.nodes.maintenance.fails metric

4 files changed, 22 insertions, 5 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 3fd15e8004a..529a280388b 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -11,6 +11,9 @@ import com.yahoo.system.ProcessExecuter;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.dockerapi.Docker;
 import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
+import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper;
+import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions;
+import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
 import com.yahoo.vespa.hosted.node.admin.util.Environment;
 import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
@@ -41,15 +44,22 @@ public class StorageMaintainer {
     private static final long intervalSec = 1000;
 
     private final Object monitor = new Object();
-    private final Environment environment;
+    private final CounterWrapper numberOfNodeAdminMaintenanceFails;
     private final Docker docker;
+    private final Environment environment;
 
     private Map<ContainerName, MetricsCache> metricsCacheByContainerName = new ConcurrentHashMap<>();
 
 
-    public StorageMaintainer(Docker docker, Environment environment) {
+    public StorageMaintainer(Docker docker, MetricReceiverWrapper metricReceiver, Environment environment) {
         this.docker = docker;
         this.environment = environment;
+
+        Dimensions dimensions = new Dimensions.Builder()
+                .add("host", HostName.getLocalhost())
+                .add("role", "docker").build();
+
+        numberOfNodeAdminMaintenanceFails = metricReceiver.declareCounter(dimensions, "nodes.maintenance.fails");
     }
 
     public Map<String, Number> updateIfNeededAndGetDiskMetricsFor(ContainerName containerName) {
@@ -278,6 +288,7 @@ public class StorageMaintainer {
             if (! result.isSuccess()) {
                 PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, executeIn);
                 logger.warning("Failed to run maintenance jobs: " + args + result);
+                numberOfNodeAdminMaintenanceFails.add();
             }
             return result;
         }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
index adc698d0a07..1f64c02e6a4 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
@@ -77,7 +77,8 @@ public class ComponentsProviderImpl implements ComponentsProvider {
                 docker,
                 metricReceiver,
                 new Environment(),
-                config.isRunningLocally() ? Optional.empty() : Optional.of(new StorageMaintainer(docker, new Environment())));
+                config.isRunningLocally() ? Optional.empty() :
+                                            Optional.of(new StorageMaintainer(docker, metricReceiver, new Environment())));
 
         if (! config.isRunningLocally()) {
             setCorePattern(docker);
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
index 59e585315f8..6107ca74752 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
@@ -1,8 +1,10 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
+import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.dockerapi.Docker;
+import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
 import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.util.Environment;
@@ -17,7 +19,7 @@ public class StorageMaintainerMock extends StorageMaintainer {
     private final CallOrderVerifier callOrderVerifier;
 
     public StorageMaintainerMock(Docker docker, Environment environment, CallOrderVerifier callOrderVerifier) {
-        super(docker, environment);
+        super(docker, new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment);
         this.callOrderVerifier = callOrderVerifier;
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
index f28086778b8..7ba48a4dbe3 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
@@ -1,5 +1,7 @@
 package com.yahoo.vespa.hosted.node.admin.maintenance;
 
+import com.yahoo.metrics.simple.MetricReceiver;
+import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.util.Environment;
 import org.junit.Rule;
 import org.junit.Test;
@@ -25,7 +27,8 @@ public class StorageMaintainerTest {
         writeNBytesToFile(folder.newFile(), writeSize);
 
         Environment environment = new Environment.Builder().build();
-        StorageMaintainer storageMaintainer = new StorageMaintainer(null, environment);
+        StorageMaintainer storageMaintainer = new StorageMaintainer(null,
+                new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment);
         long usedBytes = storageMaintainer.getDiscUsedInBytes(folder.getRoot());
         if (usedBytes * 4 < writeSize || usedBytes > writeSize * 4)
             fail("Used bytes is " + usedBytes + ", but wrote " + writeSize + " bytes, not even close.");