Added counters for number of running containers and number of fails towards docker daemon

2 files changed, 47 insertions, 22 deletions

diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
index d0a00da7eaf..5ce8684cede 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
@@ -19,9 +19,11 @@ import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory;
 import com.google.inject.Inject;
 import com.yahoo.collections.Pair;
 import com.yahoo.log.LogLevel;
-import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.system.ProcessExecuter;
 import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper;
+import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper;
+import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 
 import javax.annotation.concurrent.GuardedBy;
 import java.io.ByteArrayOutputStream;
@@ -67,13 +69,15 @@ public class DockerImpl implements Docker {
 
     final DockerClient dockerClient;
 
+    private GaugeWrapper numberOfRunningContainersGauge;
+    private CounterWrapper numberOfDockerDaemonFails;
+
     DockerImpl(final DockerClient dockerClient) {
         this.dockerClient = dockerClient;
     }
 
     @Inject
-    public DockerImpl(final DockerConfig config, MetricReceiver metricReceiver) {
-        DockerMetrics.init(metricReceiver);
+    public DockerImpl(final DockerConfig config, MetricReceiverWrapper metricReceiver) {
         JerseyDockerCmdExecFactory dockerFactory =  new JerseyDockerCmdExecFactory()
                 .withMaxPerRouteConnections(DOCKER_MAX_PER_ROUTE_CONNECTIONS)
                 .withMaxTotalConnections(DOCKER_MAX_TOTAL_CONNECTIONS)
@@ -82,9 +86,9 @@ public class DockerImpl implements Docker {
 
         RemoteApiVersion remoteApiVersion;
         try {
-             remoteApiVersion = RemoteApiVersion.parseConfig(DockerClientImpl.getInstance(
-                     buildDockerClientConfig(config).build())
-                     .withDockerCmdExecFactory(dockerFactory).versionCmd().exec().getApiVersion());
+            remoteApiVersion = RemoteApiVersion.parseConfig(DockerClientImpl.getInstance(
+                    buildDockerClientConfig(config).build())
+                    .withDockerCmdExecFactory(dockerFactory).versionCmd().exec().getApiVersion());
             logger.info("Found version of remote docker API: "+ remoteApiVersion);
             // From version 1.24 a field was removed which causes trouble with the current docker java code.
             // When this is fixed, we can remove this and do not specify version.
@@ -99,8 +103,8 @@ public class DockerImpl implements Docker {
 
         this.dockerClient = DockerClientImpl.getInstance(
                 buildDockerClientConfig(config)
-                .withApiVersion(remoteApiVersion)
-                .build())
+                        .withApiVersion(remoteApiVersion)
+                        .build())
                 .withDockerCmdExecFactory(dockerFactory);
 
         try {
@@ -108,6 +112,9 @@ public class DockerImpl implements Docker {
         } catch (Exception e) {
             throw new RuntimeException("Could not setup docker network", e);
         }
+
+        numberOfRunningContainersGauge = metricReceiver.declageGauge("number_of_running_containers");
+        numberOfDockerDaemonFails = metricReceiver.declareCounter("number_of_docker_deamon_fails");
     }
 
     static DefaultDockerClientConfig.Builder buildDockerClientConfig(DockerConfig config) {
@@ -216,6 +223,7 @@ public class DockerImpl implements Docker {
                     flatMap(image -> Arrays.stream(image.getRepoTags())).
                     anyMatch(tag -> tag.equals(dockerImage.asString()));
         } catch (DockerException e) {
+            numberOfDockerDaemonFails.add();
             throw new RuntimeException("Failed to list image name: '" + dockerImage + "'", e);
         }
     }
@@ -228,9 +236,14 @@ public class DockerImpl implements Docker {
 
     @Override
     public void connectContainerToNetwork(ContainerName containerName, String networkName) {
-        dockerClient.connectToNetworkCmd()
-                .withContainerId(containerName.asString())
-                .withNetworkId(networkName).exec();
+        try {
+            dockerClient.connectToNetworkCmd()
+                    .withContainerId(containerName.asString())
+                    .withNetworkId(networkName).exec();
+        } catch (DockerException e) {
+            numberOfDockerDaemonFails.add();
+            throw new RuntimeException("Failed to connect container to network", e);
+        }
     }
 
     @Override
@@ -255,6 +268,7 @@ public class DockerImpl implements Docker {
 
             return new ProcessResult(exitCode, new String(output.toByteArray()), new String(errors.toByteArray()));
         } catch (DockerException | InterruptedException e) {
+            numberOfDockerDaemonFails.add();
             throw new RuntimeException("Container " + containerName.asString()
                     + " failed to execute " + Arrays.toString(args), e);
         }
@@ -262,8 +276,13 @@ public class DockerImpl implements Docker {
 
     @Override
     public ContainerInfo inspectContainer(ContainerName containerName) {
-        InspectContainerResponse containerInfo = dockerClient.inspectContainerCmd(containerName.asString()).exec();
-        return new ContainerInfoImpl(containerName, containerInfo);
+        try {
+            InspectContainerResponse containerInfo = dockerClient.inspectContainerCmd(containerName.asString()).exec();
+            return new ContainerInfoImpl(containerName, containerInfo);
+        } catch (DockerException e) {
+            numberOfDockerDaemonFails.add();
+            throw new RuntimeException("Failed to get container info", e);
+        }
     }
 
     @Override
@@ -272,8 +291,9 @@ public class DockerImpl implements Docker {
         if (dockerContainer.isPresent()) {
             try {
                 dockerClient.startContainerCmd(dockerContainer.get().getId()).exec();
-                DockerMetrics.updateNumberRunningContainers(getAllManagedContainers().size());
+                numberOfRunningContainersGauge.sample(getAllManagedContainers().size());
             } catch (DockerException e) {
+                numberOfDockerDaemonFails.add();
                 throw new RuntimeException("Failed to start container", e);
             }
         }
@@ -287,6 +307,7 @@ public class DockerImpl implements Docker {
             try {
                 dockerClient.stopContainerCmd(dockerContainer.get().getId()).withTimeout(SECONDS_TO_WAIT_BEFORE_KILLING).exec();
             } catch (DockerException e) {
+                numberOfDockerDaemonFails.add();
                 throw new RuntimeException("Failed to stop container", e);
             }
         }
@@ -298,8 +319,9 @@ public class DockerImpl implements Docker {
         if (dockerContainer.isPresent()) {
             try {
                 dockerClient.removeContainerCmd(dockerContainer.get().getId()).exec();
-                DockerMetrics.updateNumberRunningContainers(getAllManagedContainers().size());
+                numberOfRunningContainersGauge.sample(getAllManagedContainers().size());
             } catch (DockerException e) {
+                numberOfDockerDaemonFails.add();
                 throw new RuntimeException("Failed to delete container", e);
             }
         }
@@ -313,6 +335,7 @@ public class DockerImpl implements Docker {
                     .flatMap(this::asContainer)
                     .collect(Collectors.toList());
         } catch (DockerException e) {
+            numberOfDockerDaemonFails.add();
             throw new RuntimeException("Could not retrieve all container", e);
         }
     }
@@ -334,6 +357,7 @@ public class DockerImpl implements Docker {
                     new ContainerName(decode(response.getName())),
                     response.getState().getRunning()));
         } catch (DockerException e) {
+            numberOfDockerDaemonFails.add();
             //TODO: do proper exception handling
             throw new RuntimeException("Failed talking to docker daemon", e);
         }
@@ -371,7 +395,12 @@ public class DockerImpl implements Docker {
 
     @Override
     public void deleteImage(final DockerImage dockerImage) {
-        dockerClient.removeImageCmd(dockerImage.asString()).exec();
+        try {
+            dockerClient.removeImageCmd(dockerImage.asString()).exec();
+        } catch (DockerException e) {
+            numberOfDockerDaemonFails.add();
+            throw new RuntimeException("Failed to delete docker image " + dockerImage.asString(), e);
+        }
     }
 
     private Map<String, Image> filterOutImagesUsedByContainers(
@@ -453,11 +482,7 @@ public class DockerImpl implements Docker {
 
     @Override
     public void deleteUnusedDockerImages(Set<DockerImage> except) {
-        try {
-            getUnusedDockerImages(except).stream().forEach(this::deleteImage);
-        } catch (DockerException e) {
-            throw new RuntimeException("Unexpected exception", e);
-        }
+        getUnusedDockerImages(except).stream().forEach(this::deleteImage);
     }
 
     private class ImagePullCallback extends PullImageResultCallback {
diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
index f8bd1fde2d2..7aa0ae9b701 100644
--- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
+++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
@@ -42,7 +42,7 @@ public class DockerTest {
             .clientKeyPath("")
             .uri("unix:///var/run/docker.sock"));
 
-    private static final DockerImpl docker = new DockerImpl(dockerConfig, MetricReceiver.nullImplementation);
+    private static final DockerImpl docker = new DockerImpl(dockerConfig, null);
     private static final DockerImage dockerImage = new DockerImage("simple-ipv6-server:Dockerfile");