diff options
author | Håkon Hallingstad <hakon@yahoo-inc.com> | 2016-08-31 13:14:13 +0200 |
---|---|---|
committer | Håkon Hallingstad <hakon@yahoo-inc.com> | 2016-09-01 12:48:00 +0200 |
commit | c8d9fb3e150cfdcffd14d96df0040c0c6a616736 (patch) | |
tree | 35ffcc06752ba7d47997f368b4e17214853efd9f /docker-api | |
parent | da7a0474414dbb50733180ac0ac52f4b1f9811b5 (diff) |
Need to figure out what to do with the tests using DockerOperations
Diffstat (limited to 'docker-api')
17 files changed, 1580 insertions, 57 deletions
diff --git a/docker-api/pom.xml b/docker-api/pom.xml index c8aa213d375..6c06481564d 100644 --- a/docker-api/pom.xml +++ b/docker-api/pom.xml @@ -24,6 +24,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application-model</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.github.docker-java</groupId> <artifactId>docker-java</artifactId> <version>3.0.3</version> @@ -43,6 +49,28 @@ <artifactId>lz4</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-junit</artifactId> + <version>2.0.0.0</version> + <scope>test</scope> + </dependency> </dependencies> <build> @@ -56,6 +84,8 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> + <source>1.8</source> + <target>1.8</target> <compilerArgs> <arg>-Xlint:all</arg> <arg>-Werror</arg> diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/docker/api/docker/DockerApi.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/docker/api/docker/DockerApi.java deleted file mode 100644 index f114adfb988..00000000000 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/docker/api/docker/DockerApi.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.yahoo.vespa.hosted.docker.api.docker; - -import com.github.dockerjava.core.DefaultDockerClientConfig; -import com.github.dockerjava.core.DockerClientImpl; -import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory; - -import com.github.dockerjava.api.DockerClient; -import com.yahoo.component.AbstractComponent; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * A class wrapping the DockerJava library for OSGI to avoid dependency problem between this library and Vespa. - * @author dybdahl - */ -public class DockerApi extends AbstractComponent { - private static final String LABEL_NAME_MANAGEDBY = "com.yahoo.vespa.managedby"; - private static final String LABEL_VALUE_MANAGEDBY = "node-admin"; - private static final Map<String, String> CONTAINER_LABELS = new HashMap<>(); - - private static final int DOCKER_MAX_PER_ROUTE_CONNECTIONS = 10; - private static final int DOCKER_MAX_TOTAL_CONNECTIONS = 100; - private static final int DOCKER_CONNECT_TIMEOUT_MILLIS = (int) TimeUnit.SECONDS.toMillis(100); - private static final int DOCKER_READ_TIMEOUT_MILLIS = (int) TimeUnit.MINUTES.toMillis(30); - - static { - CONTAINER_LABELS.put(LABEL_NAME_MANAGEDBY, LABEL_VALUE_MANAGEDBY); - } - - private final DockerClient dockerClient; - - public DockerApi() { - dockerClient = DockerClientImpl.getInstance(new DefaultDockerClientConfig.Builder() - // Talks HTTP(S) over a TCP port. The docker client library does only support tcp:// and unix:// - .withDockerHost("unix:///host/var/run/docker.sock") // Alternatively, but - // does not work due to certificate issues as if Aug 18th 2016: config.uri().replace("https", "tcp")) - .withDockerTlsVerify(false) - //.withCustomSslConfig(new VespaSSLConfig(config)) - // We can specify which version of the docker remote API to use, otherwise, use latest - // e.g. .withApiVersion("1.23") - .build()) - .withDockerCmdExecFactory( - new JerseyDockerCmdExecFactory() - .withMaxPerRouteConnections(DOCKER_MAX_PER_ROUTE_CONNECTIONS) - .withMaxTotalConnections(DOCKER_MAX_TOTAL_CONNECTIONS) - .withConnectTimeout(DOCKER_CONNECT_TIMEOUT_MILLIS) - .withReadTimeout(DOCKER_READ_TIMEOUT_MILLIS) - ); - } - - public DockerClient getDockerClient() { - return dockerClient; - } -} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java new file mode 100644 index 00000000000..1dc54311277 --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi; + +import com.yahoo.vespa.applicationmodel.HostName; + +import java.util.Objects; + +/** + * @author stiankri + */ +public class Container { + public final HostName hostname; + public final DockerImage image; + public final ContainerName name; + public final boolean isRunning; + + public Container( + final HostName hostname, + final DockerImage image, + final ContainerName containerName, + final boolean isRunning) { + this.hostname = hostname; + this.image = image; + this.name = containerName; + this.isRunning = isRunning; + } + + @Override + public boolean equals(final Object obj) { + if (!(obj instanceof Container)) { + return false; + } + final Container other = (Container) obj; + return Objects.equals(hostname, other.hostname) + && Objects.equals(image, other.image) + && Objects.equals(name, other.name) + && Objects.equals(isRunning, other.isRunning); + } + + @Override + public int hashCode() { + return Objects.hash(hostname, image, name, isRunning); + } + + @Override + public String toString() { + return "Container {" + + " hostname=" + hostname + + " image=" + image + + " name=" + name + + " isRunning=" + isRunning + + "}"; + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerInfoImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerInfoImpl.java new file mode 100644 index 00000000000..fcfad041b76 --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerInfoImpl.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi; + +import com.github.dockerjava.api.command.InspectContainerResponse; + +import java.util.Optional; + +class ContainerInfoImpl implements Docker.ContainerInfo { + + private final ContainerName containerName; + private final InspectContainerResponse inspectContainerResponse; + + ContainerInfoImpl(ContainerName containerName, InspectContainerResponse inspectContainerResponse) { + this.containerName = containerName; + this.inspectContainerResponse = inspectContainerResponse; + } + + @Override + public Optional<Integer> getPid() { + InspectContainerResponse.ContainerState state = inspectContainerResponse.getState(); + Integer containerPid = -1; + if (state.getRunning()) { + containerPid = state.getPid(); + if (containerPid == null) { + throw new RuntimeException("PID of running container " + containerName + " is null"); + } + + return Optional.of(containerPid); + } + + return Optional.empty(); + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerName.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerName.java new file mode 100644 index 00000000000..3ccd71bee6e --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerName.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi; + +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * Type-safe value wrapper for docker container names. + * + * @author bakksjo + */ +public class ContainerName { + private static final Pattern LEGAL_CONTAINER_NAME_PATTERN = Pattern.compile("^[a-zA-Z0-9-]+$"); + private final String name; + + public ContainerName(final String name) { + this.name = Objects.requireNonNull(name); + if (! LEGAL_CONTAINER_NAME_PATTERN.matcher(name).matches()) { + throw new IllegalArgumentException("Illegal container name: " + name + ". Must match " + + LEGAL_CONTAINER_NAME_PATTERN.toString()); + } + } + + public String asString() { + return name; + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public boolean equals(final Object o) { + if (!(o instanceof ContainerName)) { + return false; + } + + final ContainerName other = (ContainerName) o; + + return Objects.equals(name, other.name); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " {" + + " name=" + name + + " }"; + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java new file mode 100644 index 00000000000..f8e636c87c0 --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi; + +import com.yahoo.vespa.applicationmodel.HostName; + +import java.net.InetAddress; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +/** + * API to simplify the com.github.dockerjava API for clients, + * and to avoid OSGi exporting those classes. + */ +public interface Docker { + interface StartContainerCommand { + StartContainerCommand withLabel(String name, String value); + StartContainerCommand withEnvironment(String name, String value); + StartContainerCommand withVolume(String path, String volumePath); + StartContainerCommand withMemoryInMb(long megaBytes); + StartContainerCommand withNetworkMode(String mode); + StartContainerCommand withIpv6Address(String address); + void start(); + } + + StartContainerCommand createStartContainerCommand( + DockerImage dockerImage, + ContainerName containerName, + HostName hostName); + + void stopContainer(ContainerName containerName); + + void deleteContainer(ContainerName containerName); + + List<Container> getAllManagedContainers(); + + Optional<Container> getContainer(HostName hostname); + + CompletableFuture<DockerImage> pullImageAsync(DockerImage image); + + boolean imageIsDownloaded(DockerImage image); + + void deleteImage(DockerImage dockerImage); + + /** + * Returns the local images that are currently not in use by any container. + */ + Set<DockerImage> getUnusedDockerImages(); + + /** + * TODO: Make this function interruptible, see https://github.com/spotify/docker-client/issues/421 + * + * @param args Program arguments. args[0] must be the program filename. + * @throws RuntimeException (or some subclass thereof) on failure, including docker failure, command failure + */ + ProcessResult executeInContainer(ContainerName containerName, String... args); + + interface ContainerInfo { + /** returns Optional.empty() if not running. */ + Optional<Integer> getPid(); + } + + ContainerInfo inspectContainer(ContainerName containerName); +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImage.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImage.java new file mode 100644 index 00000000000..231eac84b38 --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImage.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi; + +import java.util.Objects; + +/** + * Type-safe value wrapper for docker image reference. + * + * @author bakksjo + */ +public class DockerImage { + private final String imageId; + + public DockerImage(final String imageId) { + this.imageId = Objects.requireNonNull(imageId); + } + + public String asString() { + return imageId; + } + + @Override + public int hashCode() { + return imageId.hashCode(); + } + + @Override + public boolean equals(final Object o) { + if (!(o instanceof DockerImage)) { + return false; + } + + final DockerImage other = (DockerImage) o; + + return Objects.equals(imageId, other.imageId); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " {" + + " imageId=" + imageId + + " }"; + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java new file mode 100644 index 00000000000..464c4fc8cac --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -0,0 +1,389 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.ExecCreateCmdResponse; +import com.github.dockerjava.api.command.ExecStartCmd; +import com.github.dockerjava.api.command.InspectContainerResponse; +import com.github.dockerjava.api.command.InspectExecResponse; +import com.github.dockerjava.api.exception.DockerClientException; +import com.github.dockerjava.api.exception.DockerException; +import com.github.dockerjava.api.model.Image; +import com.github.dockerjava.core.DefaultDockerClientConfig; +import com.github.dockerjava.core.DockerClientImpl; +import com.github.dockerjava.core.RemoteApiVersion; +import com.github.dockerjava.core.command.ExecStartResultCallback; +import com.github.dockerjava.core.command.PullImageResultCallback; +import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory; +import com.google.inject.Inject; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.applicationmodel.HostName; + +import javax.annotation.concurrent.GuardedBy; +import java.io.ByteArrayOutputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +class DockerImpl implements Docker { + + private static final Logger logger = Logger.getLogger(DockerImpl.class.getName()); + + private static final String LABEL_NAME_MANAGEDBY = "com.yahoo.vespa.managedby"; + private static final String LABEL_VALUE_MANAGEDBY = "node-admin"; + + private static final int SECONDS_TO_WAIT_BEFORE_KILLING = 10; + private static final String FRAMEWORK_CONTAINER_PREFIX = "/"; + + private static final int DOCKER_MAX_PER_ROUTE_CONNECTIONS = 10; + private static final int DOCKER_MAX_TOTAL_CONNECTIONS = 100; + private static final int DOCKER_CONNECT_TIMEOUT_MILLIS = (int) TimeUnit.SECONDS.toMillis(100); + private static final int DOCKER_READ_TIMEOUT_MILLIS = (int) TimeUnit.MINUTES.toMillis(30); + + private static final String DOCKER_CUSTOM_IP6_NETWORK_NAME = "habla"; + + private final Object monitor = new Object(); + @GuardedBy("monitor") + private final Map<DockerImage, CompletableFuture<DockerImage>> scheduledPulls = new HashMap<>(); + + final DockerClient dockerClient; + + DockerImpl(final DockerClient dockerClient) { + this.dockerClient = dockerClient; + } + + @Inject + public DockerImpl(final DockerConfig config) { + JerseyDockerCmdExecFactory dockerFactory = new JerseyDockerCmdExecFactory() + .withMaxPerRouteConnections(DOCKER_MAX_PER_ROUTE_CONNECTIONS) + .withMaxTotalConnections(DOCKER_MAX_TOTAL_CONNECTIONS) + .withConnectTimeout(DOCKER_CONNECT_TIMEOUT_MILLIS) + .withReadTimeout(DOCKER_READ_TIMEOUT_MILLIS); + + RemoteApiVersion remoteApiVersion; + try { + remoteApiVersion = RemoteApiVersion.parseConfig(DockerClientImpl.getInstance() + .withDockerCmdExecFactory(dockerFactory).versionCmd().exec().getApiVersion()); + logger.info("Found version of remote docker API: "+ remoteApiVersion); + // From version 1.24 a field was removed which causes trouble with the current docker java code. + // When this is fixed, we can remove this and do not specify version. + if (remoteApiVersion.isGreaterOrEqual(RemoteApiVersion.VERSION_1_24)) { + remoteApiVersion = RemoteApiVersion.VERSION_1_23; + logger.info("Found version 1.24 or newer of remote API, using 1.23."); + } + } catch (Exception e) { + logger.log(LogLevel.ERROR, "Failed when trying to figure out remote API version of docker, using 1.23", e); + remoteApiVersion = RemoteApiVersion.VERSION_1_23; + } + + // DockerClientImpl.getInstance().infoCmd().exec().getServerVersion(); + this.dockerClient = DockerClientImpl.getInstance(new DefaultDockerClientConfig.Builder() + .withDockerHost(config.uri()) + .withApiVersion(remoteApiVersion) + .build()) + .withDockerCmdExecFactory(dockerFactory); + } + + @Override + public CompletableFuture<DockerImage> pullImageAsync(final DockerImage image) { + final CompletableFuture<DockerImage> completionListener; + synchronized (monitor) { + if (scheduledPulls.containsKey(image)) { + return scheduledPulls.get(image); + } + completionListener = new CompletableFuture<>(); + scheduledPulls.put(image, completionListener); + } + dockerClient.pullImageCmd(image.asString()).exec(new ImagePullCallback(image)); + return completionListener; + } + + private CompletableFuture<DockerImage> removeScheduledPoll(final DockerImage image) { + synchronized (monitor) { + return scheduledPulls.remove(image); + } + } + + /** + * Check if a given image is already in the local registry + */ + @Override + public boolean imageIsDownloaded(final DockerImage dockerImage) { + try { + List<Image> images = dockerClient.listImagesCmd().withShowAll(true).exec(); + return images.stream(). + flatMap(image -> Arrays.stream(image.getRepoTags())). + anyMatch(tag -> tag.equals(dockerImage.asString())); + } catch (DockerException e) { + throw new RuntimeException("Failed to list image name: '" + dockerImage + "'", e); + } + } + + @Override + public StartContainerCommand createStartContainerCommand(DockerImage image, ContainerName name, HostName hostName) { + return new StartContainerCommandImpl(dockerClient, image, name, hostName) + .withLabel(LABEL_NAME_MANAGEDBY, LABEL_VALUE_MANAGEDBY); + } + + @Override + public ProcessResult executeInContainer(ContainerName containerName, String... args) { + assert args.length >= 1; + try { + final ExecCreateCmdResponse response = dockerClient.execCreateCmd(containerName.asString()) + .withCmd(args) + .withAttachStdout(true) + .withAttachStderr(true) + .exec(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + ByteArrayOutputStream errors = new ByteArrayOutputStream(); + ExecStartCmd execStartCmd = dockerClient.execStartCmd(response.getId()); + execStartCmd.exec(new ExecStartResultCallback(output, errors)).awaitCompletion(); + + final InspectExecResponse state = dockerClient.inspectExecCmd(execStartCmd.getExecId()).exec(); + assert !state.isRunning(); + Integer exitCode = state.getExitCode(); + assert exitCode != null; + + return new ProcessResult(exitCode, new String(output.toByteArray()), new String(errors.toByteArray())); + } catch (DockerException | InterruptedException e) { + throw new RuntimeException("Container " + containerName.asString() + + " failed to execute " + Arrays.toString(args), e); + } + } + + @Override + public ContainerInfo inspectContainer(ContainerName containerName) { + InspectContainerResponse containerInfo = dockerClient.inspectContainerCmd(containerName.asString()).exec(); + return new ContainerInfoImpl(containerName, containerInfo); + } + + @Override + public void stopContainer(final ContainerName containerName) { + Optional<com.github.dockerjava.api.model.Container> dockerContainer = getContainerFromName(containerName, true); + if (dockerContainer.isPresent()) { + try { + dockerClient.stopContainerCmd(dockerContainer.get().getId()).withTimeout(SECONDS_TO_WAIT_BEFORE_KILLING).exec(); + } catch (DockerException e) { + throw new RuntimeException("Failed to stop container", e); + } + } + } + + @Override + public void deleteContainer(ContainerName containerName) { + Optional<com.github.dockerjava.api.model.Container> dockerContainer = getContainerFromName(containerName, true); + if (dockerContainer.isPresent()) { + try { + dockerClient.removeContainerCmd(dockerContainer.get().getId()).exec(); + } catch (DockerException e) { + throw new RuntimeException("Failed to delete container", e); + } + } + } + + @Override + public List<Container> getAllManagedContainers() { + try { + return dockerClient.listContainersCmd().withShowAll(true).exec().stream() + .filter(this::isManaged) + .flatMap(this::asContainer) + .collect(Collectors.toList()); + } catch (DockerException e) { + throw new RuntimeException("Could not retrieve all container", e); + } + } + + @Override + public Optional<Container> getContainer(HostName hostname) { + // TODO Don't rely on getAllManagedContainers + return getAllManagedContainers().stream() + .filter(c -> Objects.equals(hostname, c.hostname)) + .findFirst(); + } + + private Stream<Container> asContainer(com.github.dockerjava.api.model.Container dockerClientContainer) { + try { + final InspectContainerResponse response = dockerClient.inspectContainerCmd(dockerClientContainer.getId()).exec(); + return Stream.of(new Container( + new HostName(response.getConfig().getHostName()), + new DockerImage(dockerClientContainer.getImage()), + new ContainerName(decode(response.getName())), + response.getState().getRunning())); + } catch (DockerException e) { + //TODO: do proper exception handling + throw new RuntimeException("Failed talking to docker daemon", e); + } + } + + + private Optional<com.github.dockerjava.api.model.Container> getContainerFromName( + final ContainerName containerName, final boolean alsoGetStoppedContainers) { + try { + return dockerClient.listContainersCmd().withShowAll(alsoGetStoppedContainers).exec().stream() + .filter(this::isManaged) + .filter(container -> matchName(container, containerName.asString())) + .findFirst(); + } catch (DockerException e) { + throw new RuntimeException("Failed to get container from name", e); + } + } + + private boolean isManaged(final com.github.dockerjava.api.model.Container container) { + final Map<String, String> labels = container.getLabels(); + if (labels == null) { + return false; + } + + return LABEL_VALUE_MANAGEDBY.equals(labels.get(LABEL_NAME_MANAGEDBY)); + } + + private boolean matchName(com.github.dockerjava.api.model.Container container, String targetName) { + return Arrays.stream(container.getNames()).anyMatch(encodedName -> decode(encodedName).equals(targetName)); + } + + private String decode(String encodedContainerName) { + return encodedContainerName.substring(FRAMEWORK_CONTAINER_PREFIX.length()); + } + + @Override + public void deleteImage(final DockerImage dockerImage) { + dockerClient.removeImageCmd(dockerImage.asString()).exec(); + } + + @Override + public Set<DockerImage> getUnusedDockerImages() { + // Description of concepts and relationships: + // - a docker image has an id, and refers to its parent image (if any) by image id. + // - a docker image may, in addition to id, have multiple tags, but each tag identifies exactly one image. + // - a docker container refers to its image (exactly one) either by image id or by image tag. + // What this method does to find images considered unused, is build a tree of dependencies + // (e.g. container->tag->image->image) and identify image nodes whose only children (if any) are leaf tags. + // In other words, an image node with no children, or only tag children having no children themselves is unused. + // An image node with an image child is considered used. + // An image node with a container child is considered used. + // An image node with a tag child with a container child is considered used. + try { + final Map<String, DockerObject> objects = new HashMap<>(); + final Map<String, String> dependencies = new HashMap<>(); + + // Populate maps with images (including tags) and their dependencies (parents). + for (Image image : dockerClient.listImagesCmd().withShowAll(true).exec()) { + objects.put(image.getId(), new DockerObject(image.getId(), DockerObjectType.IMAGE)); + if (image.getParentId() != null && !image.getParentId().isEmpty()) { + dependencies.put(image.getId(), image.getParentId()); + } + for (String tag : image.getRepoTags()) { + objects.put(tag, new DockerObject(tag, DockerObjectType.IMAGE_TAG)); + dependencies.put(tag, image.getId()); + } + } + + // Populate maps with containers and their dependency to the image they run on. + for (com.github.dockerjava.api.model.Container container : dockerClient.listContainersCmd().withShowAll(true).exec()) { + objects.put(container.getId(), new DockerObject(container.getId(), DockerObjectType.CONTAINER)); + dependencies.put(container.getId(), container.getImage()); + } + + // Now update every object with its dependencies. + dependencies.forEach((fromId, toId) -> { + Optional.ofNullable(objects.get(toId)) + .ifPresent(obj -> obj.addDependee(objects.get(fromId))); + }); + + // Find images that are not in use (i.e. leafs not used by any containers). + return objects.values().stream() + .filter(dockerObject -> dockerObject.type == DockerObjectType.IMAGE) + .filter(dockerObject -> !dockerObject.isInUse()) + .map(obj -> obj.id) + .map(DockerImage::new) + .collect(Collectors.toSet()); + } catch (DockerException e) { + throw new RuntimeException("Unexpected exception", e); + } + } + + // Helper enum for calculating which images are unused. + private enum DockerObjectType { + IMAGE_TAG, IMAGE, CONTAINER + } + + // Helper class for calculating which images are unused. + private static class DockerObject { + public final String id; + public final DockerObjectType type; + private final List<DockerObject> dependees = new LinkedList<>(); + + public DockerObject(final String id, final DockerObjectType type) { + this.id = id; + this.type = type; + } + + public boolean isInUse() { + if (type == DockerObjectType.CONTAINER) { + return true; + } + + if (dependees.isEmpty()) { + return false; + } + + if (type == DockerObjectType.IMAGE) { + if (dependees.stream().anyMatch(obj -> obj.type == DockerObjectType.IMAGE)) { + return true; + } + } + + return dependees.stream().anyMatch(DockerObject::isInUse); + } + + public void addDependee(final DockerObject dockerObject) { + dependees.add(dockerObject); + } + + @Override + public String toString() { + return "DockerObject {" + + " id=" + id + + " type=" + type.name().toLowerCase() + + " inUse=" + isInUse() + + " dependees=" + dependees.stream().map(obj -> obj.id).collect(Collectors.toList()) + + " }"; + } + } + + private class ImagePullCallback extends PullImageResultCallback { + private final DockerImage dockerImage; + + ImagePullCallback(DockerImage dockerImage) { + this.dockerImage = dockerImage; + } + + @Override + public void onError(Throwable throwable) { + removeScheduledPoll(dockerImage).completeExceptionally(throwable); + } + + + @Override + public void onComplete() { + if (imageIsDownloaded(dockerImage)) { + removeScheduledPoll(dockerImage).complete(dockerImage); + } else { + removeScheduledPoll(dockerImage).completeExceptionally( + new DockerClientException("Could not download image: " + dockerImage)); + } + } + } +}
\ No newline at end of file diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ProcessResult.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ProcessResult.java new file mode 100644 index 00000000000..52796a694ce --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ProcessResult.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi; + +import java.util.Objects; + +public class ProcessResult { + private final int exitStatus; + private final String output; + private final String errors; + + public ProcessResult(int exitStatus, String output, String errors) { + this.exitStatus = exitStatus; + this.output = output; + this.errors = errors; + } + + public boolean isSuccess() { return exitStatus == 0; } + public int getExitStatus() { return exitStatus; } + + public String getOutput() { return output; } + + public String getErrors() { return errors; } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ProcessResult)) return false; + ProcessResult other = (ProcessResult) o; + return Objects.equals(exitStatus, other.exitStatus) + && Objects.equals(output, other.output) + && Objects.equals(errors, other.errors); + } + + @Override + public int hashCode() { + return Objects.hash(exitStatus, output, errors); + } + + @Override + public String toString() { + return "ProcessResult {" + + " exitStatus=" + exitStatus + + " output=" + output + + " errors=" + errors + + " }"; + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/StartContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/StartContainerCommandImpl.java new file mode 100644 index 00000000000..e03bab805fe --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/StartContainerCommandImpl.java @@ -0,0 +1,139 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.command.CreateContainerResponse; +import com.github.dockerjava.api.exception.DockerException; +import com.github.dockerjava.api.model.Bind; +import com.yahoo.vespa.applicationmodel.HostName; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +class StartContainerCommandImpl implements Docker.StartContainerCommand { + + private final DockerClient docker; + private final DockerImage dockerImage; + private final ContainerName containerName; + private final HostName hostName; + private final Map<String, String> labels = new HashMap<>(); + private final List<String> environmentAssignments = new ArrayList<>(); + private final List<String> volumeBindSpecs = new ArrayList<>(); + + private Optional<Long> memoryInB = Optional.empty(); + private Optional<String> networkMode = Optional.empty(); + private Optional<String> ipv6Address = Optional.empty(); + + StartContainerCommandImpl(DockerClient docker, + DockerImage dockerImage, + ContainerName containerName, + HostName hostName) { + this.docker = docker; + this.dockerImage = dockerImage; + this.containerName = containerName; + this.hostName = hostName; + } + + @Override + public Docker.StartContainerCommand withLabel(String name, String value) { + assert name.indexOf("=") == -1; + labels.put(name, value); + return this; + } + + @Override + public Docker.StartContainerCommand withEnvironment(String name, String value) { + assert name.indexOf('=') == -1; + environmentAssignments.add(name + "=" + value); + return this; + } + + @Override + public Docker.StartContainerCommand withVolume(String path, String volumePath) { + assert path.indexOf(':') == -1; + volumeBindSpecs.add(path + ":" + volumePath); + return this; + } + + @Override + public Docker.StartContainerCommand withMemoryInMb(long megaBytes) { + memoryInB = Optional.of(megaBytes * 1024 * 1024); + return this; + } + + @Override + public Docker.StartContainerCommand withNetworkMode(String mode) { + networkMode = Optional.of(mode); + return this; + } + + @Override + public Docker.StartContainerCommand withIpv6Address(String address) { + ipv6Address = Optional.of(address); + return this; + } + + @Override + public void start() { + CreateContainerCmd command = createCreateContainerCmd(); + + try { + CreateContainerResponse response = command.exec(); + docker.startContainerCmd(response.getId()).exec(); + } catch (DockerException e) { + throw new RuntimeException("Failed to start container " + containerName.asString(), e); + } + } + + private CreateContainerCmd createCreateContainerCmd() { + List<Bind> volumeBinds = volumeBindSpecs.stream().map(Bind::parse).collect(Collectors.toList()); + + CreateContainerCmd containerCmd = docker + .createContainerCmd(dockerImage.asString()) + .withName(containerName.asString()) + .withHostName(hostName.s()) + .withLabels(labels) + .withEnv(environmentAssignments) + .withBinds(volumeBinds); + + if (memoryInB.isPresent()) containerCmd = containerCmd.withMemory(memoryInB.get()); + if (networkMode.isPresent()) containerCmd = containerCmd.withNetworkMode(networkMode.get()); + if (ipv6Address.isPresent()) containerCmd = containerCmd.withIpv6Address(ipv6Address.get()); + + return containerCmd; + } + + /** Maps ("--env", {"A", "B", "C"}) to "--env A --env B --env C ". */ + private String toRepeatedOption(String option, List<String> optionValues) { + StringBuilder builder = new StringBuilder(); + optionValues.stream().forEach(optionValue -> builder.append(option + " " + optionValue + " ")); + return builder.toString(); + } + + private String toOptionalOption(String option, Optional<?> value) { + return value.isPresent() ? option + " " + value.get() + " " : ""; + } + + /** Make toString() print the equivalent arguments to 'docker run' */ + @Override + public String toString() { + + List<String> labelList = labels.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()).collect(Collectors.toList()); + + return "--name " + containerName + " " + + "--hostname " + hostName + " " + + toRepeatedOption("--label", labelList) + + toRepeatedOption("--env", environmentAssignments) + + toRepeatedOption("--volume", volumeBindSpecs) + + toOptionalOption("--memory", memoryInB) + + toOptionalOption("--net", networkMode) + + toOptionalOption("--ip6", ipv6Address) + + dockerImage; + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/VespaSSLConfig.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/VespaSSLConfig.java new file mode 100644 index 00000000000..84b528f6750 --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/VespaSSLConfig.java @@ -0,0 +1,229 @@ +package com.yahoo.vespa.hosted.dockerapi; + +import com.github.dockerjava.api.exception.DockerClientException; +import com.github.dockerjava.core.SSLConfig; +import org.bouncycastle.asn1.ASN1ObjectIdentifier; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openssl.PEMKeyPair; +import org.bouncycastle.openssl.PEMParser; +import org.glassfish.jersey.SslConfigurator; + +import javax.net.ssl.SSLContext; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.KeyFactory; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.Security; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Objects.requireNonNull; + + +/** + * This class is based off {@link com.github.dockerjava.core.LocalDirectorySSLConfig}, but with the ability to + * specify path to each of the certificates instead of directory path. Additionally it includes + * {@link com.github.dockerjava.core.util.CertificateUtils} because of version conflict of with + * com.google.code.findbugs.annotations + */ +public class VespaSSLConfig implements SSLConfig { + private final DockerConfig config; + + public VespaSSLConfig(DockerConfig config) { + this.config = config; + } + + @Override + public SSLContext getSSLContext() { + try { + Security.addProvider(new BouncyCastleProvider()); + + // properties acrobatics not needed for java > 1.6 + String httpProtocols = System.getProperty("https.protocols"); + System.setProperty("https.protocols", "TLSv1"); + SslConfigurator sslConfig = SslConfigurator.newInstance(true); + if (httpProtocols != null) { + System.setProperty("https.protocols", httpProtocols); + } + + String keypem = new String(Files.readAllBytes(Paths.get(config.clientKeyPath()))); + String certpem = new String(Files.readAllBytes(Paths.get(config.clientCertPath()))); + String capem = new String(Files.readAllBytes(Paths.get(config.caCertPath()))); + + sslConfig.keyStore(createKeyStore(keypem, certpem)); + sslConfig.keyStorePassword("docker"); + sslConfig.trustStore(createTrustStore(capem)); + + return sslConfig.createSSLContext(); + } catch (Exception e) { + throw new DockerClientException(e.getMessage(), e); + } + } + + public static KeyStore createKeyStore(final String keypem, final String certpem) throws NoSuchAlgorithmException, + InvalidKeySpecException, IOException, CertificateException, KeyStoreException { + PrivateKey privateKey = loadPrivateKey(keypem); + requireNonNull(privateKey); + List<Certificate> privateCertificates = loadCertificates(certpem); + + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null); + + keyStore.setKeyEntry("docker", + privateKey, + "docker".toCharArray(), + privateCertificates.toArray(new Certificate[privateCertificates.size()]) + ); + + return keyStore; + } + + /** + * from "cert.pem" String + */ + private static List<Certificate> loadCertificates(final String certpem) throws IOException, + CertificateException { + final StringReader certReader = new StringReader(certpem); + try (BufferedReader reader = new BufferedReader(certReader)) { + return loadCertificates(reader); + } + } + + /** + * "cert.pem" from reader + */ + private static List<Certificate> loadCertificates(final Reader reader) throws IOException, + CertificateException { + try (PEMParser pemParser = new PEMParser(reader)) { + List<Certificate> certificates = new ArrayList<>(); + + JcaX509CertificateConverter certificateConverter = new JcaX509CertificateConverter().setProvider("BC"); + Object certObj = pemParser.readObject(); + + if (certObj instanceof X509CertificateHolder) { + X509CertificateHolder certificateHolder = (X509CertificateHolder) certObj; + certificates.add(certificateConverter.getCertificate(certificateHolder)); + } + + return certificates; + } + } + + + /** + * Return private key ("key.pem") from Reader + */ + private static PrivateKey loadPrivateKey(final Reader reader) throws IOException, NoSuchAlgorithmException, + InvalidKeySpecException { + try (PEMParser pemParser = new PEMParser(reader)) { + Object readObject = pemParser.readObject(); + while (readObject != null) { + if (readObject instanceof PEMKeyPair) { + PEMKeyPair pemKeyPair = (PEMKeyPair) readObject; + PrivateKey privateKey = guessKey(pemKeyPair.getPrivateKeyInfo().getEncoded()); + if (privateKey != null) { + return privateKey; + } + } else if (readObject instanceof PrivateKeyInfo) { + PrivateKeyInfo privateKeyInfo = (PrivateKeyInfo) readObject; + PrivateKey privateKey = guessKey(privateKeyInfo.getEncoded()); + if (privateKey != null) { + return privateKey; + } + } else if (readObject instanceof ASN1ObjectIdentifier) { + // no idea how it can be used + final ASN1ObjectIdentifier asn1ObjectIdentifier = (ASN1ObjectIdentifier) readObject; + } + + readObject = pemParser.readObject(); + } + } + + return null; + } + + private static PrivateKey guessKey(byte[] encodedKey) throws NoSuchAlgorithmException { + //no way to know, so iterate + for (String guessFactory : new String[]{"RSA", "ECDSA"}) { + try { + KeyFactory factory = KeyFactory.getInstance(guessFactory); + + PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(encodedKey); + return factory.generatePrivate(privateKeySpec); + } catch (InvalidKeySpecException ignore) { + } + } + + return null; + } + + /** + * Return KeyPair from "key.pem" + */ + private static PrivateKey loadPrivateKey(final String keypem) throws IOException, NoSuchAlgorithmException, + InvalidKeySpecException { + try (StringReader certReader = new StringReader(keypem); + BufferedReader reader = new BufferedReader(certReader)) { + return loadPrivateKey(reader); + } + } + + /** + * "ca.pem" from String + */ + public static KeyStore createTrustStore(String capem) throws IOException, CertificateException, + KeyStoreException, NoSuchAlgorithmException { + try (Reader certReader = new StringReader(capem)) { + return createTrustStore(certReader); + } + } + + /** + * "ca.pem" from Reader + */ + public static KeyStore createTrustStore(final Reader certReader) throws IOException, CertificateException, + KeyStoreException, NoSuchAlgorithmException { + try (PEMParser pemParser = new PEMParser(certReader)) { + X509CertificateHolder certificateHolder = (X509CertificateHolder) pemParser.readObject(); + Certificate caCertificate = new JcaX509CertificateConverter() + .setProvider("BC") + .getCertificate(certificateHolder); + + KeyStore trustStore = KeyStore.getInstance("JKS"); + trustStore.load(null); + trustStore.setCertificateEntry("ca", caCertificate); + + return trustStore; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + VespaSSLConfig that = (VespaSSLConfig) o; + + return config.equals(that.config); + + } + + @Override + public int hashCode() { + return config.hashCode(); + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/docker/api/docker/package-info.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/package-info.java index 004507fb58f..08a10f3e70d 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/docker/api/docker/package-info.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/package-info.java @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -package com.yahoo.vespa.hosted.docker.api.docker; +package com.yahoo.vespa.hosted.dockerapi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/docker-api/src/main/resources/configdefinitions/docker.def b/docker-api/src/main/resources/configdefinitions/docker.def new file mode 100644 index 00000000000..c0173d1530b --- /dev/null +++ b/docker-api/src/main/resources/configdefinitions/docker.def @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.dockerapi + +caCertPath string +clientCertPath string +clientKeyPath string +uri string default = "tcp://127.0.0.1:2376" diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/ContainerNameTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/ContainerNameTest.java new file mode 100644 index 00000000000..0522f14a06b --- /dev/null +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/ContainerNameTest.java @@ -0,0 +1,39 @@ +package com.yahoo.vespa.hosted.dockerapi; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author valerijf + */ +public class ContainerNameTest { + @Test + public void testAlphanumericalContainerName() { + String name = "container123"; + ContainerName containerName = new ContainerName(name); + assertEquals(containerName.asString(), name); + } + + @Test + public void testAlphanumericalWithDashContainerName() { + String name = "container-123"; + ContainerName containerName = new ContainerName(name); + assertEquals(containerName.asString(), name); + } + + @Test(expected=IllegalArgumentException.class) + public void testAlphanumericalWithSlashContainerName() { + new ContainerName("container/123"); + } + + @Test(expected=IllegalArgumentException.class) + public void testEmptyContainerName() { + new ContainerName(""); + } + + @Test(expected=NullPointerException.class) + public void testNullContainerName() { + new ContainerName(null); + } +} diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java new file mode 100644 index 00000000000..d400147c169 --- /dev/null +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java @@ -0,0 +1,254 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.ExecCreateCmd; +import com.github.dockerjava.api.command.ExecCreateCmdResponse; +import com.github.dockerjava.api.command.ExecStartCmd; +import com.github.dockerjava.api.command.InspectExecCmd; +import com.github.dockerjava.api.command.InspectExecResponse; +import com.github.dockerjava.api.command.ListContainersCmd; +import com.github.dockerjava.api.command.ListImagesCmd; +import com.github.dockerjava.api.model.Image; +import com.github.dockerjava.core.command.ExecStartResultCallback; +import org.junit.Test; +import org.mockito.Matchers; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author tonytv + */ +public class DockerImplTest { + @Test + public void testExecuteCompletes() throws Exception { + final String containerId = "container-id"; + final String[] command = new String[] {"/bin/ls", "-l"}; + final String execId = "exec-id"; + final int exitCode = 3; + + final DockerClient dockerClient = mock(DockerClient.class); + + final ExecCreateCmdResponse response = mock(ExecCreateCmdResponse.class); + when(response.getId()).thenReturn(execId); + + final ExecCreateCmd execCreateCmd = mock(ExecCreateCmd.class); + when(dockerClient.execCreateCmd(any(String.class))).thenReturn(execCreateCmd); + when(execCreateCmd.withCmd(Matchers.<String>anyVararg())).thenReturn(execCreateCmd); + when(execCreateCmd.withAttachStdout(any(Boolean.class))).thenReturn(execCreateCmd); + when(execCreateCmd.withAttachStderr(any(Boolean.class))).thenReturn(execCreateCmd); + when(execCreateCmd.exec()).thenReturn(response); + + final ExecStartCmd execStartCmd = mock(ExecStartCmd.class); + when(dockerClient.execStartCmd(any(String.class))).thenReturn(execStartCmd); + when(execStartCmd.exec(any(ExecStartResultCallback.class))).thenReturn(mock(ExecStartResultCallback.class)); + + final InspectExecCmd inspectExecCmd = mock(InspectExecCmd.class); + final InspectExecResponse state = mock(InspectExecResponse.class); + when(dockerClient.inspectExecCmd(any(String.class))).thenReturn(inspectExecCmd); + when(inspectExecCmd.exec()).thenReturn(state); + when(state.isRunning()).thenReturn(false); + when(state.getExitCode()).thenReturn(exitCode); + + final Docker docker = new DockerImpl(dockerClient); + final ProcessResult result = docker.executeInContainer(new ContainerName(containerId), command); + assertThat(result.getExitStatus(), is(exitCode)); + } + + @Test + public void noImagesMeansNoUnusedImages() throws Exception { + ImageGcTester + .withExistingImages() + .expectUnusedImages(); + } + + @Test + public void singleImageWithoutContainersIsUnused() throws Exception { + ImageGcTester + .withExistingImages(new ImageBuilder("image-1")) + .expectUnusedImages("image-1"); + } + + @Test + public void singleImageWithContainerIsUsed() throws Exception { + ImageGcTester + .withExistingImages(ImageBuilder.forId("image-1")) + .andExistingContainers(ContainerBuilder.forId("container-1").withImage("image-1")) + .expectUnusedImages(); + } + + @Test + public void onlyLeafImageIsUnused() throws Exception { + ImageGcTester + .withExistingImages( + ImageBuilder.forId("parent-image"), + ImageBuilder.forId("leaf-image").withParentId("parent-image")) + .expectUnusedImages("leaf-image"); + } + + @Test + public void multipleUnusedImagesAreIdentified() throws Exception { + ImageGcTester + .withExistingImages( + ImageBuilder.forId("image-1"), + ImageBuilder.forId("image-2")) + .expectUnusedImages("image-1", "image-2"); + } + + @Test + public void multipleUnusedLeavesAreIdentified() throws Exception { + ImageGcTester + .withExistingImages( + ImageBuilder.forId("parent-image"), + ImageBuilder.forId("image-1").withParentId("parent-image"), + ImageBuilder.forId("image-2").withParentId("parent-image")) + .expectUnusedImages("image-1", "image-2"); + } + + @Test + public void unusedLeafWithUsedSiblingIsIdentified() throws Exception { + ImageGcTester + .withExistingImages( + ImageBuilder.forId("parent-image"), + ImageBuilder.forId("image-1").withParentId("parent-image"), + ImageBuilder.forId("image-2").withParentId("parent-image")) + .andExistingContainers(ContainerBuilder.forId("vespa-node-1").withImage("image-1")) + .expectUnusedImages("image-2"); + } + + @Test + public void containerCanReferToImageByTag() throws Exception { + ImageGcTester + .withExistingImages(ImageBuilder.forId("image-1").withTag("vespa-6")) + .andExistingContainers(ContainerBuilder.forId("vespa-node-1").withImage("vespa-6")) + .expectUnusedImages(); + } + + @Test + public void taggedImageWithNoContainersIsUnused() throws Exception { + ImageGcTester + .withExistingImages(ImageBuilder.forId("image-1").withTag("vespa-6")) + .expectUnusedImages("image-1"); + } + + private static class ImageGcTester { + private final List<Image> existingImages; + private List<com.github.dockerjava.api.model.Container> existingContainers = Collections.emptyList(); + + private ImageGcTester(final List<Image> images) { + this.existingImages = images; + } + + public static ImageGcTester withExistingImages(final ImageBuilder... images) { + final List<Image> existingImages = Arrays.stream(images) + .map(ImageBuilder::toImage) + .collect(Collectors.toList()); + return new ImageGcTester(existingImages); + } + + public ImageGcTester andExistingContainers(final ContainerBuilder... containers) { + this.existingContainers = Arrays.stream(containers) + .map(ContainerBuilder::toContainer) + .collect(Collectors.toList()); + return this; + } + + public void expectUnusedImages(final String... imageIds) throws Exception { + final DockerClient dockerClient = mock(DockerClient.class); + final Docker docker = new DockerImpl(dockerClient); + final ListImagesCmd listImagesCmd = mock(ListImagesCmd.class); + final ListContainersCmd listContainersCmd = mock(ListContainersCmd.class); + + when(dockerClient.listImagesCmd()).thenReturn(listImagesCmd); + when(listImagesCmd.withShowAll(true)).thenReturn(listImagesCmd); + when(listImagesCmd.exec()).thenReturn(existingImages); + + when(dockerClient.listContainersCmd()).thenReturn(listContainersCmd); + when(listContainersCmd.withShowAll(true)).thenReturn(listContainersCmd); + when(listContainersCmd.exec()).thenReturn(existingContainers); + + final Set<DockerImage> expectedUnusedImages = Arrays.stream(imageIds) + .map(DockerImage::new) + .collect(Collectors.toSet()); + assertThat( + docker.getUnusedDockerImages(), + is(expectedUnusedImages)); + + } + } + + /** + * Serializes object to a JSON string using Jackson, then deserializes it to an instance of toClass + * (again using Jackson). This can be used to create Jackson classes with no public constructors. + * @throws IllegalArgumentException if Jackson fails to serialize or deserialize. + */ + private static <T> T createFrom(Class<T> toClass, Object object) throws IllegalArgumentException { + final String serialized; + try { + serialized = new ObjectMapper().writeValueAsString(object); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Failed to serialize object " + object + " to " + + toClass + " with Jackson: " + e, e); + } + try { + return new ObjectMapper().readValue(serialized, toClass); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to convert " + serialized + " to " + + toClass + " with Jackson: " + e, e); + } + } + + // Workaround for Image class that can't be instantiated directly in Java (instantiate via Jackson instead). + private static class ImageBuilder { + // Json property names must match exactly the property names in the Image class. + @JsonProperty("Id") + private final String id; + + @JsonProperty("ParentId") + private String parentId; + + @JsonProperty("RepoTags") + private final List<String> repoTags = new LinkedList<>(); + + private ImageBuilder(String id) { this.id = id; } + + public static ImageBuilder forId(String id) { return new ImageBuilder(id); } + public ImageBuilder withParentId(String parentId) { this.parentId = parentId; return this; } + public ImageBuilder withTag(String tag) { this.repoTags.add(tag); return this; } + + public Image toImage() { return createFrom(Image.class, this); } + } + + // Workaround for Container class that can't be instantiated directly in Java (instantiate via Jackson instead). + private static class ContainerBuilder { + // Json property names must match exactly the property names in the Container class. + @JsonProperty("Id") + private final String id; + + @JsonProperty("Image") + private String image; + + private ContainerBuilder(String id) { this.id = id; } + private static ContainerBuilder forId(final String id) { return new ContainerBuilder(id); } + public ContainerBuilder withImage(String image) { this.image = image; return this; } + + public com.github.dockerjava.api.model.Container toContainer() { + return createFrom(com.github.dockerjava.api.model.Container.class, this); + } + } +} diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java new file mode 100644 index 00000000000..50c8492d0b4 --- /dev/null +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java @@ -0,0 +1,177 @@ +package com.yahoo.vespa.hosted.dockerapi; + +import com.github.dockerjava.api.model.Network; +import com.github.dockerjava.core.command.BuildImageResultCallback; +import com.yahoo.vespa.applicationmodel.HostName; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.util.Collections; +import java.util.concurrent.ExecutionException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author valerijf + */ +public class DockerTest { + /** + * To run these tests: + * 1. Remove Ignore annotations + * 2. Change ownership of docker.sock + * $ sudo chown <your username> /var/run/docker.sock + * 3. (Temporary) Manually create the docker network used by DockerImpl by running: + * $ sudo docker network create --ipv6 --gateway=<your local IPv6 address> --subnet=fe80::1/16 habla + * 4. (Temporary) Manually build docker test image. Inside src/test/resources/simple-ipv6-server run: + * $ sudo docker build -t "simple-ipv6-server:Dockerfile" . + * 5. (Temporary) Comment out setup() and shutdown() + */ + private static final DockerConfig dockerConfig = new DockerConfig(new DockerConfig.Builder() + .caCertPath("") // Temporary setting it to empty as this field is required, in the future + .clientCertPath("") // DockerConfig should be rewritten and probably moved to docker-api module + .clientKeyPath("") + .uri("unix:///var/run/docker.sock")); + + private static final DockerImpl docker = new DockerImpl(dockerConfig); + private static final DockerImage dockerImage = new DockerImage("simple-ipv6-server:Dockerfile"); + + + @Ignore + @Test + public void testDockerImagePull() throws ExecutionException, InterruptedException { + DockerImage dockerImage = new DockerImage("busybox:1.24.0"); + + // Pull the image and wait for the pull to complete + docker.pullImageAsync(dockerImage).get(); + + // Translate the human readable ID to sha256-hash ID that is returned by getUnusedDockerImages() + DockerImage targetImage = new DockerImage(docker.dockerClient.inspectImageCmd(dockerImage.asString()).exec().getId()); + assertTrue("Image: " + dockerImage + " should be unused", docker.getUnusedDockerImages().contains(targetImage)); + + // Remove the image + docker.deleteImage(dockerImage); + assertFalse("Failed to delete " + dockerImage.asString() + " image", docker.imageIsDownloaded(dockerImage)); + } + + @Ignore + @Test + public void testDockerNetworking() throws InterruptedException, ExecutionException, IOException { + HostName hostName1 = new HostName("docker10.test.yahoo.com"); + HostName hostName2 = new HostName("docker11.test.yahoo.com"); + ContainerName containerName1 = new ContainerName("test-container-1"); + ContainerName containerName2 = new ContainerName("test-container-2"); + InetAddress inetAddress1 = Inet6Address.getByName("fe80::10"); + InetAddress inetAddress2 = Inet6Address.getByName("fe80::11"); + + // TODO: Use new Docker API. + // docker.startContainer(dockerImage, hostName1, containerName1, inetAddress1, 0, 0, 0); + // docker.startContainer(dockerImage, hostName2, containerName2, inetAddress2, 0, 0, 0); + + try { + testReachabilityFromHost(containerName1, inetAddress1); + testReachabilityFromHost(containerName2, inetAddress2); + + String[] curlFromNodeToNode = new String[]{"curl", "-g", "http://[" + inetAddress2.getHostAddress() + "%eth0]/ping"}; + while (! docker.executeInContainer(containerName1, curlFromNodeToNode).isSuccess()) { + Thread.sleep(20); + } + ProcessResult result = docker.executeInContainer(containerName1, curlFromNodeToNode); + assertTrue("Could not reach " + containerName2.asString() + " from " + containerName1.asString(), + result.getOutput().equals("pong\n")); + } finally { + docker.stopContainer(containerName1); + docker.deleteContainer(containerName1); + + docker.stopContainer(containerName2); + docker.deleteContainer(containerName2); + } + } + + private void testReachabilityFromHost(ContainerName containerName, InetAddress target) throws IOException, InterruptedException { + String[] curlNodeFromHost = {"curl", "-g", "http://[" + target.getHostAddress() + "%" + getInterfaceName() + "]/ping"}; + while (!exec(curlNodeFromHost).equals("pong\n")) { + Thread.sleep(20); + } + assertTrue("Could not reach " + containerName.asString() + " from host", exec(curlNodeFromHost).equals("pong\n")); + } + + + /** + * Returns IPv6 address of on the "docker0" interface that can be reached by the containers + */ + private static String getLocalIPv6Address() throws SocketException { + return Collections.list(NetworkInterface.getNetworkInterfaces()).stream() + .filter(networkInterface -> networkInterface.getDisplayName().equals("docker0")) + .flatMap(i -> Collections.list(i.getInetAddresses()).stream()) + .filter(ip -> ip instanceof Inet6Address && ip.isLinkLocalAddress()) + .findFirst().orElseThrow(RuntimeException::new) + .getHostAddress().split("%")[0]; + } + + /** + * Returns the display name of the bridge used by our custom docker network. This is used for routing in the + * network tests. The bridge is assumed to be the only IPv6 interface starting with "br-" + */ + private static String getInterfaceName() throws SocketException { + return Collections.list(NetworkInterface.getNetworkInterfaces()).stream() + .filter(networkInterface -> networkInterface.getDisplayName().startsWith("br-") && + networkInterface.getInterfaceAddresses().stream() + .anyMatch(ip -> ip.getAddress() instanceof Inet6Address)) + .findFirst().orElseThrow(RuntimeException::new).getDisplayName(); + } + + /** + * Synchronously executes a system process and returns its stdout. Based of {@link com.yahoo.system.ProcessExecuter} + * but could not be reused because of import errors. + */ + private static String exec(String[] command) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder(command); + StringBuilder ret = new StringBuilder(); + + Process p = pb.start(); + InputStream is = p.getInputStream(); + while (true) { + int b = is.read(); + if (b==-1) break; + ret.append((char) b); + } + + p.waitFor(); + p.destroy(); + + return ret.toString(); + } + + @Before + public void setup() throws IOException, ExecutionException, InterruptedException { + // Build the image locally + File dockerFilePath = new File("src/test/resources/simple-ipv6-server"); + docker.dockerClient + .buildImageCmd(dockerFilePath) + .withTag(dockerImage.asString()).exec(new BuildImageResultCallback()).awaitCompletion(); + + // Create a temporary network + Network.Ipam ipam = new Network.Ipam().withConfig(new Network.Ipam.Config() + .withSubnet("fe80::1/16").withGateway(getLocalIPv6Address())); + // TODO: This needs to match the network name in DockerOperations!? + docker.dockerClient.createNetworkCmd().withDriver("bridge").withName("habla") + .withIpam(ipam).exec(); + } + + @After + public void shutdown() { + // Remove the network we created earlier + // TODO: This needs to match the network name in DockerOperations!? + docker.dockerClient.removeNetworkCmd("habla").exec(); + } +} diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/ProcessResultTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/ProcessResultTest.java new file mode 100644 index 00000000000..b28d3f3c29a --- /dev/null +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/ProcessResultTest.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class ProcessResultTest { + @Test + public void testBasicProperties() throws Exception { + ProcessResult processResult = new ProcessResult(0, "foo", "bar"); + assertThat(processResult.getExitStatus(), is(0)); + assertThat(processResult.getOutput(), is("foo")); + assertThat(processResult.isSuccess(), is(true)); + } + + @Test + public void testSuccessFails() throws Exception { + ProcessResult processResult = new ProcessResult(1, "foo", "bar"); + assertThat(processResult.isSuccess(), is(false)); + } +} |