diff options
author | Haakon Dybdahl <dybis@users.noreply.github.com> | 2016-10-17 09:27:34 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-10-17 09:27:34 +0200 |
commit | bc74da749a5809324ed96b48a061efa600d798a1 (patch) | |
tree | 0596cd0019ab9b2985dc9e115bbd9d2f9fb85ae8 /docker-api/src | |
parent | 187f7558d18ca1cdfb4c1c8c2c57af82ea0bdf1a (diff) | |
parent | 154389a2f90d0399314d3f0c29fc065a697c4c0a (diff) |
Merge pull request #862 from yahoo/dybdahl/docker-network-on-mac
Create a network test for Mac.
Diffstat (limited to 'docker-api/src')
5 files changed, 165 insertions, 57 deletions
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index 609896a01bb..8856c1770d1 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -25,6 +25,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { private final Map<String, String> labels = new HashMap<>(); private final List<String> environmentAssignments = new ArrayList<>(); private final List<String> volumeBindSpecs = new ArrayList<>(); + private final List<String> commands = new ArrayList<>(); private Optional<Long> memoryInB = Optional.empty(); private Optional<String> networkMode = Optional.empty(); @@ -49,6 +50,13 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { } @Override + public Docker.CreateContainerCommand withCmd(String name) { + commands.add(name); + return this; + } + + + @Override public Docker.CreateContainerCommand withEnvironment(String name, String value) { assert name.indexOf('=') == -1; environmentAssignments.add(name + "=" + value); @@ -105,6 +113,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { .withEnv(environmentAssignments) .withBinds(volumeBinds); + if (! commands.isEmpty()) containerCmd = containerCmd.withCmd(commands); if (memoryInB.isPresent()) containerCmd = containerCmd.withMemory(memoryInB.get()); if (networkMode.isPresent()) containerCmd = containerCmd.withNetworkMode(networkMode.get()); if (ipv4Address.isPresent()) containerCmd = containerCmd.withIpv4Address(ipv4Address.get()); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index 2a17b139d2b..4877f103fd4 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -20,6 +20,8 @@ public interface Docker { CreateContainerCommand withMemoryInMb(long megaBytes); CreateContainerCommand withNetworkMode(String mode); CreateContainerCommand withIpAddress(InetAddress address); + CreateContainerCommand withCmd(String name); + void create(); } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 5e5185bc6be..b886bffa6de 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.dockerapi; import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CreateNetworkCmd; import com.github.dockerjava.api.command.ExecCreateCmdResponse; import com.github.dockerjava.api.command.ExecStartCmd; import com.github.dockerjava.api.command.InspectContainerResponse; @@ -35,7 +36,9 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.Inet6Address; import java.net.InetAddress; +import java.net.SocketException; import java.net.URI; +import java.net.UnknownHostException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -85,24 +88,30 @@ public class DockerImpl implements Docker { this.dockerClient = dockerClient; } - // For testing - DockerImpl(final DockerConfig config) { - hackAroundPullImageDueToJerseyConflicts = false; - // Fail fast - initDockerConnection(config, 100 /* connect timeout millis */, false /* fallback to 1.23 on errors */); - setMetrics(new MetricReceiverWrapper(MetricReceiver.nullImplementation)); + public DockerImpl( + final DockerConfig config, + boolean fallbackTo123OnErrors, + boolean trySetupNetowork, + Optional<String> subnetwork, + boolean hackAroundPullImageDueToJerseyConflicts, + int dockerConnectTimeoutMillis, + MetricReceiverWrapper metricReceiverWrapper) { + this.hackAroundPullImageDueToJerseyConflicts = hackAroundPullImageDueToJerseyConflicts; + initDockerConnection(config, dockerConnectTimeoutMillis, fallbackTo123OnErrors); + setMetrics(metricReceiverWrapper); + if (trySetupNetowork) setupDockerNetworkIfNeeded(subnetwork); } @Inject public DockerImpl(final DockerConfig config, MetricReceiverWrapper metricReceiver) { - hackAroundPullImageDueToJerseyConflicts = true; - initDockerConnection(config, DOCKER_CONNECT_TIMEOUT_MILLIS, true /* fallback to 1.23 on errors */); - try { - setupDockerNetworkIfNeeded(); - } catch (Exception e) { - throw new RuntimeException("Could not setup docker network", e); - } - setMetrics(metricReceiver); + this( + config, + true, /* fallback to 1.23 on errors */ + true, /* try setup netowork */ + Optional.empty(), /* isLinux */ + true, /* hackAroundPullImageDueToJerseyConflicts */ + DOCKER_CONNECT_TIMEOUT_MILLIS /* dockerConnectTimeoutMillis */, + metricReceiver); } static DefaultDockerClientConfig.Builder buildDockerClientConfig(DockerConfig config) { @@ -120,32 +129,57 @@ public class DockerImpl implements Docker { return dockerConfigBuilder; } - private void setupDockerNetworkIfNeeded() throws IOException, InterruptedException { + private void setupDockerNetworkIfNeeded(Optional<String> subnetwork) { if (! dockerClient.listNetworksCmd().withNameFilter(DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).exec().isEmpty()) return; // Use IPv6 address if there is a mix of IP4 and IPv6 by taking the longest address. - List<InetAddress> hostAddresses = Arrays.asList(InetAddress.getAllByName(com.yahoo.net.HostName.getLocalhost())); + final List<InetAddress> hostAddresses; + try { + hostAddresses = Arrays.asList(InetAddress.getAllByName(HostName.getLocalhost())); + } catch (UnknownHostException e) { + throw new RuntimeException("Could not get hostAddresses", e); + } + InetAddress hostAddress = Collections.max(hostAddresses, (o1, o2) -> o1.getAddress().length - o2.getAddress().length); - NetworkAddressInterface networkAddressInterface = DockerNetworkCreator.getInterfaceForAddress(hostAddress); + final NetworkAddressInterface networkAddressInterface; + try { + networkAddressInterface = DockerNetworkCreator.getInterfaceForAddress(hostAddress); + } catch (SocketException|UnknownHostException e) { + throw new RuntimeException("Could not get getInterfaceForAddress for " + hostAddress, e); + } boolean isIPv6 = networkAddressInterface.interfaceAddress.getAddress() instanceof Inet6Address; + final String defaultGateway; + try { + defaultGateway = DockerNetworkCreator.getDefaultGatewayLinux(isIPv6).getHostAddress(); + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not find default gateway, isIpv6 is " + isIPv6, e); + } + Network.Ipam ipam = new Network.Ipam().withConfig(new Network.Ipam.Config() .withSubnet(hostAddress.getHostAddress() + "/" + networkAddressInterface.interfaceAddress.getNetworkPrefixLength()) - .withGateway(DockerNetworkCreator.getDefaultGateway(isIPv6).getHostAddress())); + .withGateway(defaultGateway)); + + if (subnetwork.isPresent()) { + ipam = new Network.Ipam().withConfig(new Network.Ipam.Config().withSubnet(subnetwork.get())); + } Map<String, String> dockerNetworkOptions = new HashMap<>(); dockerNetworkOptions.put("parent", networkAddressInterface.networkInterface.getDisplayName()); dockerNetworkOptions.put("macvlan_mode", "bridge"); - dockerClient.createNetworkCmd() - .withName(DOCKER_CUSTOM_MACVLAN_NETWORK_NAME) - .withDriver("macvlan") - .withEnableIpv6(isIPv6) + CreateNetworkCmd createNetworkCmd = dockerClient.createNetworkCmd() .withIpam(ipam) - .withOptions(dockerNetworkOptions) - .exec(); + .withName(DOCKER_CUSTOM_MACVLAN_NETWORK_NAME); + if (! subnetwork.isPresent()) { + createNetworkCmd + .withDriver("macvlan") + .withEnableIpv6(isIPv6) + .withOptions(dockerNetworkOptions); + } + createNetworkCmd.exec(); } @Override diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerNetworkCreator.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerNetworkCreator.java index 32982d1c16c..03f8375e02d 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerNetworkCreator.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerNetworkCreator.java @@ -14,7 +14,7 @@ import java.util.Collections; * @author valerijf */ class DockerNetworkCreator { - static InetAddress getDefaultGateway(boolean ipv6) throws IOException, InterruptedException { + static InetAddress getDefaultGatewayLinux(boolean ipv6) throws IOException, InterruptedException { String command = ipv6 ? "route -A inet6 -n | grep 'UG[ \t]' | awk '{print $2}'" : "route -n | grep 'UG[ \t]' | awk '{print $2}'"; diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java index a77776f53dc..e41074d9354 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java @@ -3,15 +3,22 @@ package com.yahoo.vespa.hosted.dockerapi; import com.github.dockerjava.api.model.Network; import com.github.dockerjava.core.command.BuildImageResultCallback; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import org.apache.commons.io.IOUtils; +import org.junit.Ignore; import org.junit.Test; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.net.Inet6Address; import java.net.InetAddress; +import java.net.URL; import java.util.HashSet; import java.util.List; -import java.util.Set; +import java.util.Optional; import java.util.concurrent.ExecutionException; import static junit.framework.TestCase.fail; @@ -25,13 +32,23 @@ import static org.junit.Assume.assumeTrue; * Class for testing full integration with docker daemon, requires running daemon. To run these tests: * * MAC: - * 1. Install Docker Toolbox, and start it (Docker Quick Start Terminal) (you can close terminal window afterwards) + * 1. Install Docker Toolbox, and start it (Docker Quickstart Terminal) (you can close terminal window afterwards) + * 2. For network test, we need to make docker containers visible for Mac: sudo route add 172.0.0.0/8 192.168.99.100 * 2. Run tests from IDE/mvn. * + + * TIPS: + * For cleaning up your local docker machine (DON'T DO THIS ON PROD) + * docker stop $(docker ps -a -q) + * docker rm $(docker ps -a -q) + * * @author valerijf * @author dybdahl */ public class DockerTest { + public static final String SUBNET_CONTAINER = "172.18.7.2/24"; + public static final String IP_ADDRESS_FOR_CONTAINER = "172.18.7.32"; + private DockerImpl docker; private static final OS operatingSystem = getSystemOS(); private static final String prefix = "/Users/" + System.getProperty("user.name") + "/.docker/machine/machines/default/"; @@ -51,10 +68,11 @@ public class DockerTest { assertThat(containers.isEmpty(), is(true)); } + // It is ignored since it is a bit slow and unstable, at least on Mac. + @Ignore @Test public void testDockerImagePull() throws ExecutionException, InterruptedException { assumeTrue(dockerDaemonIsPresent()); - DockerImpl docker = new DockerImpl(dockerConfig); docker.getAllManagedContainers(); DockerImage dockerImage = new DockerImage("busybox:1.24.0"); @@ -74,33 +92,76 @@ public class DockerTest { } @Test - public void testCreateDeleteImageCreateDeleteContainer() throws IOException, InterruptedException, ExecutionException { + public void testContainerCycle() throws IOException, InterruptedException, ExecutionException { assumeTrue(dockerDaemonIsPresent()); - createDockerImage(docker); - ContainerName containerName = new ContainerName("foo"); - docker.stopContainer(containerName); - docker.deleteContainer(containerName); - assertThat(docker.getAllManagedContainers().isEmpty(), is(true)); - docker.createContainerCommand(dockerImage, containerName, "hostName1").create(); - List<Container> containers = docker.getAllManagedContainers(); - assertThat(containers.size(), is(1)); - docker.deleteContainer(containerName); + DockerImage busyBoxImage = new DockerImage("busybox:1.24.0"); + // Pull the image and wait for the pull to complete + System.out.print("Pulling image."); + docker.pullImageAsync(busyBoxImage).get(); + System.out.println("... done"); - docker.pullImageAsync(dockerImage).get(); + ContainerName containerName = new ContainerName("DummyWebServer"); - // Translate the human readable ID to sha256-hash ID that is returned by getUnusedDockerImages() - DockerImage targetImage = new DockerImage(docker.dockerClient.inspectImageCmd(dockerImage.asString()).exec().getId()); - Set<DockerImage> except = new HashSet<>(); - List<DockerImage> x = docker.getUnusedDockerImages(except); + try { // Stop any old container + docker.stopContainer(containerName); + System.out.println("Stopped an old container with same name"); + } catch (Exception e) { + System.out.println("Did not manage to stop container " + e.getMessage()); + } + try { // Delete any remaining container + docker.deleteContainer(containerName); + System.out.println("Deleted an old container with same name"); + } catch (Exception e) { + System.out.println("Did not manage to delete container " + e.getMessage()); + } - // Remove the image - docker.deleteImage(dockerImage); - List<DockerImage> y = docker.getUnusedDockerImages(except); + int webPortForContainer = 6342; - assertFalse("Failed to delete " + dockerImage.asString() + " image", docker.imageIsDownloaded(dockerImage)); + InetAddress inetAddress1 = Inet6Address.getByName(IP_ADDRESS_FOR_CONTAINER); + docker.createContainerCommand(busyBoxImage, containerName, "hostName1") + .withNetworkMode("vespa-macvlan") + .withIpAddress(inetAddress1) + .withCmd("sleep").withCmd("55555").create(); + + System.out.print("Starting container with sleep running."); + docker.startContainer(containerName); + System.out.println("..done"); + + + // TODO: Get the container to start with httpd instead of hacking it here. + new Thread(() -> { + try { + // This call is blocking + docker.executeInContainer(containerName, "/bin/sh", "-c", "/bin/httpd -f -p " + webPortForContainer).getOutput(); + } catch (RuntimeException e) { + // ignore, fails on shut down + } + }).start(); + + boolean success = false; + for (int x = 0; x < 600; x++) { + try { + URL url = new URL("http://" + IP_ADDRESS_FOR_CONTAINER + ":" + webPortForContainer); + InputStream is = url.openStream(); + String containerServer = IOUtils.toString(is); + assertThat(containerServer, is("sdf")); + } catch (FileNotFoundException e) { + System.out.println("Managed to talk to web server in container."); + success = true; + break; + } catch (Throwable t) { + System.err.println(t.getMessage()); + Thread.sleep(100); + } + } + docker.stopContainer(containerName); + docker.deleteContainer(containerName); + // Do not bother, it takes forever to load it again.. docker.deleteImage(busyBoxImage); + assertThat(success, is(true)); } + @Test public void testDockerNetworking() throws InterruptedException, ExecutionException, IOException { assumeTrue(dockerDaemonIsPresent()); @@ -142,14 +203,6 @@ public class DockerTest { } } - private void createDockerTestNetworkIfNeeded() { - if (! docker.dockerClient.listNetworksCmd().withNameFilter(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).exec().isEmpty()) return; - - Network.Ipam ipam = new Network.Ipam().withConfig(new Network.Ipam.Config().withSubnet("172.18.0.0/16")); - docker.dockerClient.createNetworkCmd() - .withName(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withDriver("bridge").withIpam(ipam).exec(); - } - private boolean dockerDaemonIsPresent() { if (docker != null) return true; if (operatingSystem == OS.Unsupported) { @@ -158,8 +211,7 @@ public class DockerTest { } try { - docker = new DockerImpl(dockerConfig); - createDockerTestNetworkIfNeeded(); + setDocker(); return true; } catch (Exception e) { System.out.println("Please install Docker Toolbox and start Docker Quick Start Terminal once, ignoring test."); @@ -168,6 +220,17 @@ public class DockerTest { } } + private void setDocker() { + docker = new DockerImpl( + dockerConfig, + false, /* fallback to 1.23 on errors */ + false, /* try setup netowork */ + operatingSystem == OS.Mac_OS_X ? Optional.of(SUBNET_CONTAINER) : Optional.empty(), /* subnetwork */ + false, /* hackAroundPullImageDueToJerseyConflicts */ + 100 /* dockerConnectTimeoutMillis */, + new MetricReceiverWrapper(MetricReceiver.nullImplementation)); + } + private void testReachabilityFromHost(ContainerName containerName, InetAddress target) throws IOException, InterruptedException { String[] curlNodeFromHost = {"curl", "-g", "http://" + target.getHostAddress() + "/ping"}; while (!exec(curlNodeFromHost).equals("pong\n")) { |