From c3d7af641984aaa81cd15567a282e35c9e5291ba Mon Sep 17 00:00:00 2001 From: freva Date: Thu, 26 Jan 2017 14:54:59 +0100 Subject: Add docker.nodes.maintenance.fails metric --- .../hosted/node/admin/maintenance/StorageMaintainer.java | 15 +++++++++++++-- .../node/admin/provider/ComponentsProviderImpl.java | 3 ++- .../admin/integrationTests/StorageMaintainerMock.java | 4 +++- .../node/admin/maintenance/StorageMaintainerTest.java | 5 ++++- 4 files changed, 22 insertions(+), 5 deletions(-) (limited to 'node-admin/src') diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 3fd15e8004a..529a280388b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -11,6 +11,9 @@ import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; +import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; @@ -41,15 +44,22 @@ public class StorageMaintainer { private static final long intervalSec = 1000; private final Object monitor = new Object(); - private final Environment environment; + private final CounterWrapper numberOfNodeAdminMaintenanceFails; private final Docker docker; + private final Environment environment; private Map metricsCacheByContainerName = new ConcurrentHashMap<>(); - public StorageMaintainer(Docker docker, Environment environment) { + public StorageMaintainer(Docker docker, MetricReceiverWrapper metricReceiver, Environment environment) { this.docker = docker; this.environment = environment; + + Dimensions dimensions = new Dimensions.Builder() + .add("host", HostName.getLocalhost()) + .add("role", "docker").build(); + + numberOfNodeAdminMaintenanceFails = metricReceiver.declareCounter(dimensions, "nodes.maintenance.fails"); } public Map updateIfNeededAndGetDiskMetricsFor(ContainerName containerName) { @@ -278,6 +288,7 @@ public class StorageMaintainer { if (! result.isSuccess()) { PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, executeIn); logger.warning("Failed to run maintenance jobs: " + args + result); + numberOfNodeAdminMaintenanceFails.add(); } return result; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java index adc698d0a07..1f64c02e6a4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java @@ -77,7 +77,8 @@ public class ComponentsProviderImpl implements ComponentsProvider { docker, metricReceiver, new Environment(), - config.isRunningLocally() ? Optional.empty() : Optional.of(new StorageMaintainer(docker, new Environment()))); + config.isRunningLocally() ? Optional.empty() : + Optional.of(new StorageMaintainer(docker, metricReceiver, new Environment()))); if (! config.isRunningLocally()) { setCorePattern(docker); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java index 59e585315f8..6107ca74752 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java @@ -1,8 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.util.Environment; @@ -17,7 +19,7 @@ public class StorageMaintainerMock extends StorageMaintainer { private final CallOrderVerifier callOrderVerifier; public StorageMaintainerMock(Docker docker, Environment environment, CallOrderVerifier callOrderVerifier) { - super(docker, environment); + super(docker, new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment); this.callOrderVerifier = callOrderVerifier; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index f28086778b8..7ba48a4dbe3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -1,5 +1,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.util.Environment; import org.junit.Rule; import org.junit.Test; @@ -25,7 +27,8 @@ public class StorageMaintainerTest { writeNBytesToFile(folder.newFile(), writeSize); Environment environment = new Environment.Builder().build(); - StorageMaintainer storageMaintainer = new StorageMaintainer(null, environment); + StorageMaintainer storageMaintainer = new StorageMaintainer(null, + new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment); long usedBytes = storageMaintainer.getDiscUsedInBytes(folder.getRoot()); if (usedBytes * 4 < writeSize || usedBytes > writeSize * 4) fail("Used bytes is " + usedBytes + ", but wrote " + writeSize + " bytes, not even close."); -- cgit v1.2.3