diff options
author | valerijf <valerijf@yahoo-inc.com> | 2016-09-27 09:44:35 +0200 |
---|---|---|
committer | valerijf <valerijf@yahoo-inc.com> | 2016-09-29 09:45:46 +0200 |
commit | e35d3bdec5338dfae5883f23bd8814bde5bdcc15 (patch) | |
tree | 69dc83372186d4c46e2ab72045361be986120532 /docker-api | |
parent | 5da079ece24d5c4d456cef15906a31fb75336263 (diff) |
Added counters for number of running containers and number of fails towards docker daemon
Diffstat (limited to 'docker-api')
-rw-r--r-- | docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java | 67 | ||||
-rw-r--r-- | docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java | 2 |
2 files changed, 47 insertions, 22 deletions
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index d0a00da7eaf..5ce8684cede 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -19,9 +19,11 @@ import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory; import com.google.inject.Inject; import com.yahoo.collections.Pair; import com.yahoo.log.LogLevel; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import javax.annotation.concurrent.GuardedBy; import java.io.ByteArrayOutputStream; @@ -67,13 +69,15 @@ public class DockerImpl implements Docker { final DockerClient dockerClient; + private GaugeWrapper numberOfRunningContainersGauge; + private CounterWrapper numberOfDockerDaemonFails; + DockerImpl(final DockerClient dockerClient) { this.dockerClient = dockerClient; } @Inject - public DockerImpl(final DockerConfig config, MetricReceiver metricReceiver) { - DockerMetrics.init(metricReceiver); + public DockerImpl(final DockerConfig config, MetricReceiverWrapper metricReceiver) { JerseyDockerCmdExecFactory dockerFactory = new JerseyDockerCmdExecFactory() .withMaxPerRouteConnections(DOCKER_MAX_PER_ROUTE_CONNECTIONS) .withMaxTotalConnections(DOCKER_MAX_TOTAL_CONNECTIONS) @@ -82,9 +86,9 @@ public class DockerImpl implements Docker { RemoteApiVersion remoteApiVersion; try { - remoteApiVersion = RemoteApiVersion.parseConfig(DockerClientImpl.getInstance( - buildDockerClientConfig(config).build()) - .withDockerCmdExecFactory(dockerFactory).versionCmd().exec().getApiVersion()); + remoteApiVersion = RemoteApiVersion.parseConfig(DockerClientImpl.getInstance( + buildDockerClientConfig(config).build()) + .withDockerCmdExecFactory(dockerFactory).versionCmd().exec().getApiVersion()); logger.info("Found version of remote docker API: "+ remoteApiVersion); // From version 1.24 a field was removed which causes trouble with the current docker java code. // When this is fixed, we can remove this and do not specify version. @@ -99,8 +103,8 @@ public class DockerImpl implements Docker { this.dockerClient = DockerClientImpl.getInstance( buildDockerClientConfig(config) - .withApiVersion(remoteApiVersion) - .build()) + .withApiVersion(remoteApiVersion) + .build()) .withDockerCmdExecFactory(dockerFactory); try { @@ -108,6 +112,9 @@ public class DockerImpl implements Docker { } catch (Exception e) { throw new RuntimeException("Could not setup docker network", e); } + + numberOfRunningContainersGauge = metricReceiver.declageGauge("number_of_running_containers"); + numberOfDockerDaemonFails = metricReceiver.declareCounter("number_of_docker_deamon_fails"); } static DefaultDockerClientConfig.Builder buildDockerClientConfig(DockerConfig config) { @@ -216,6 +223,7 @@ public class DockerImpl implements Docker { flatMap(image -> Arrays.stream(image.getRepoTags())). anyMatch(tag -> tag.equals(dockerImage.asString())); } catch (DockerException e) { + numberOfDockerDaemonFails.add(); throw new RuntimeException("Failed to list image name: '" + dockerImage + "'", e); } } @@ -228,9 +236,14 @@ public class DockerImpl implements Docker { @Override public void connectContainerToNetwork(ContainerName containerName, String networkName) { - dockerClient.connectToNetworkCmd() - .withContainerId(containerName.asString()) - .withNetworkId(networkName).exec(); + try { + dockerClient.connectToNetworkCmd() + .withContainerId(containerName.asString()) + .withNetworkId(networkName).exec(); + } catch (DockerException e) { + numberOfDockerDaemonFails.add(); + throw new RuntimeException("Failed to connect container to network", e); + } } @Override @@ -255,6 +268,7 @@ public class DockerImpl implements Docker { return new ProcessResult(exitCode, new String(output.toByteArray()), new String(errors.toByteArray())); } catch (DockerException | InterruptedException e) { + numberOfDockerDaemonFails.add(); throw new RuntimeException("Container " + containerName.asString() + " failed to execute " + Arrays.toString(args), e); } @@ -262,8 +276,13 @@ public class DockerImpl implements Docker { @Override public ContainerInfo inspectContainer(ContainerName containerName) { - InspectContainerResponse containerInfo = dockerClient.inspectContainerCmd(containerName.asString()).exec(); - return new ContainerInfoImpl(containerName, containerInfo); + try { + InspectContainerResponse containerInfo = dockerClient.inspectContainerCmd(containerName.asString()).exec(); + return new ContainerInfoImpl(containerName, containerInfo); + } catch (DockerException e) { + numberOfDockerDaemonFails.add(); + throw new RuntimeException("Failed to get container info", e); + } } @Override @@ -272,8 +291,9 @@ public class DockerImpl implements Docker { if (dockerContainer.isPresent()) { try { dockerClient.startContainerCmd(dockerContainer.get().getId()).exec(); - DockerMetrics.updateNumberRunningContainers(getAllManagedContainers().size()); + numberOfRunningContainersGauge.sample(getAllManagedContainers().size()); } catch (DockerException e) { + numberOfDockerDaemonFails.add(); throw new RuntimeException("Failed to start container", e); } } @@ -287,6 +307,7 @@ public class DockerImpl implements Docker { try { dockerClient.stopContainerCmd(dockerContainer.get().getId()).withTimeout(SECONDS_TO_WAIT_BEFORE_KILLING).exec(); } catch (DockerException e) { + numberOfDockerDaemonFails.add(); throw new RuntimeException("Failed to stop container", e); } } @@ -298,8 +319,9 @@ public class DockerImpl implements Docker { if (dockerContainer.isPresent()) { try { dockerClient.removeContainerCmd(dockerContainer.get().getId()).exec(); - DockerMetrics.updateNumberRunningContainers(getAllManagedContainers().size()); + numberOfRunningContainersGauge.sample(getAllManagedContainers().size()); } catch (DockerException e) { + numberOfDockerDaemonFails.add(); throw new RuntimeException("Failed to delete container", e); } } @@ -313,6 +335,7 @@ public class DockerImpl implements Docker { .flatMap(this::asContainer) .collect(Collectors.toList()); } catch (DockerException e) { + numberOfDockerDaemonFails.add(); throw new RuntimeException("Could not retrieve all container", e); } } @@ -334,6 +357,7 @@ public class DockerImpl implements Docker { new ContainerName(decode(response.getName())), response.getState().getRunning())); } catch (DockerException e) { + numberOfDockerDaemonFails.add(); //TODO: do proper exception handling throw new RuntimeException("Failed talking to docker daemon", e); } @@ -371,7 +395,12 @@ public class DockerImpl implements Docker { @Override public void deleteImage(final DockerImage dockerImage) { - dockerClient.removeImageCmd(dockerImage.asString()).exec(); + try { + dockerClient.removeImageCmd(dockerImage.asString()).exec(); + } catch (DockerException e) { + numberOfDockerDaemonFails.add(); + throw new RuntimeException("Failed to delete docker image " + dockerImage.asString(), e); + } } private Map<String, Image> filterOutImagesUsedByContainers( @@ -453,11 +482,7 @@ public class DockerImpl implements Docker { @Override public void deleteUnusedDockerImages(Set<DockerImage> except) { - try { - getUnusedDockerImages(except).stream().forEach(this::deleteImage); - } catch (DockerException e) { - throw new RuntimeException("Unexpected exception", e); - } + getUnusedDockerImages(except).stream().forEach(this::deleteImage); } private class ImagePullCallback extends PullImageResultCallback { diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java index f8bd1fde2d2..7aa0ae9b701 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java @@ -42,7 +42,7 @@ public class DockerTest { .clientKeyPath("") .uri("unix:///var/run/docker.sock")); - private static final DockerImpl docker = new DockerImpl(dockerConfig, MetricReceiver.nullImplementation); + private static final DockerImpl docker = new DockerImpl(dockerConfig, null); private static final DockerImage dockerImage = new DockerImage("simple-ipv6-server:Dockerfile"); |