Merge pull request #862 from yahoo/dybdahl/docker-network-on-mac

Create a network test for Mac.

5 files changed, 165 insertions, 57 deletions

diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
index 609896a01bb..8856c1770d1 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
@@ -25,6 +25,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
     private final Map<String, String> labels = new HashMap<>();
     private final List<String> environmentAssignments = new ArrayList<>();
     private final List<String> volumeBindSpecs = new ArrayList<>();
+    private final List<String> commands = new ArrayList<>();
 
     private Optional<Long> memoryInB = Optional.empty();
     private Optional<String> networkMode = Optional.empty();
@@ -49,6 +50,13 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
     }
 
     @Override
+    public Docker.CreateContainerCommand withCmd(String name) {
+        commands.add(name);
+        return this;
+    }
+
+
+    @Override
     public Docker.CreateContainerCommand withEnvironment(String name, String value) {
         assert name.indexOf('=') == -1;
         environmentAssignments.add(name + "=" + value);
@@ -105,6 +113,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
                 .withEnv(environmentAssignments)
                 .withBinds(volumeBinds);
 
+        if (! commands.isEmpty()) containerCmd = containerCmd.withCmd(commands);
         if (memoryInB.isPresent()) containerCmd = containerCmd.withMemory(memoryInB.get());
         if (networkMode.isPresent()) containerCmd = containerCmd.withNetworkMode(networkMode.get());
         if (ipv4Address.isPresent()) containerCmd = containerCmd.withIpv4Address(ipv4Address.get());
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
index 2a17b139d2b..4877f103fd4 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
@@ -20,6 +20,8 @@ public interface Docker {
         CreateContainerCommand withMemoryInMb(long megaBytes);
         CreateContainerCommand withNetworkMode(String mode);
         CreateContainerCommand withIpAddress(InetAddress address);
+        CreateContainerCommand withCmd(String name);
+
         void create();
     }
 
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
index 5e5185bc6be..b886bffa6de 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.dockerapi;
 
 import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CreateNetworkCmd;
 import com.github.dockerjava.api.command.ExecCreateCmdResponse;
 import com.github.dockerjava.api.command.ExecStartCmd;
 import com.github.dockerjava.api.command.InspectContainerResponse;
@@ -35,7 +36,9 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.net.Inet6Address;
 import java.net.InetAddress;
+import java.net.SocketException;
 import java.net.URI;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -85,24 +88,30 @@ public class DockerImpl implements Docker {
         this.dockerClient = dockerClient;
     }
 
-    // For testing
-    DockerImpl(final DockerConfig config) {
-        hackAroundPullImageDueToJerseyConflicts = false;
-        // Fail fast
-        initDockerConnection(config, 100 /* connect timeout millis */, false /* fallback to 1.23 on errors */);
-        setMetrics(new MetricReceiverWrapper(MetricReceiver.nullImplementation));
+    public DockerImpl(
+            final DockerConfig config,
+            boolean fallbackTo123OnErrors,
+            boolean trySetupNetowork,
+            Optional<String> subnetwork,
+            boolean hackAroundPullImageDueToJerseyConflicts,
+            int dockerConnectTimeoutMillis,
+            MetricReceiverWrapper metricReceiverWrapper) {
+        this.hackAroundPullImageDueToJerseyConflicts = hackAroundPullImageDueToJerseyConflicts;
+        initDockerConnection(config, dockerConnectTimeoutMillis, fallbackTo123OnErrors);
+        setMetrics(metricReceiverWrapper);
+        if (trySetupNetowork) setupDockerNetworkIfNeeded(subnetwork);
     }
 
     @Inject
     public DockerImpl(final DockerConfig config, MetricReceiverWrapper metricReceiver) {
-        hackAroundPullImageDueToJerseyConflicts = true;
-        initDockerConnection(config, DOCKER_CONNECT_TIMEOUT_MILLIS, true /* fallback to 1.23 on errors */);
-        try {
-            setupDockerNetworkIfNeeded();
-        } catch (Exception e) {
-            throw new RuntimeException("Could not setup docker network", e);
-        }
-        setMetrics(metricReceiver);
+        this(
+                config,
+                true, /* fallback to 1.23 on errors */
+                true, /* try setup netowork */
+                Optional.empty(), /* isLinux */
+                true, /* hackAroundPullImageDueToJerseyConflicts */
+                DOCKER_CONNECT_TIMEOUT_MILLIS /* dockerConnectTimeoutMillis */,
+                metricReceiver);
     }
 
     static DefaultDockerClientConfig.Builder buildDockerClientConfig(DockerConfig config) {
@@ -120,32 +129,57 @@ public class DockerImpl implements Docker {
         return dockerConfigBuilder;
     }
 
-    private void setupDockerNetworkIfNeeded() throws IOException, InterruptedException {
+    private void setupDockerNetworkIfNeeded(Optional<String> subnetwork)  {
         if (! dockerClient.listNetworksCmd().withNameFilter(DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).exec().isEmpty()) return;
 
         // Use IPv6 address if there is a mix of IP4 and IPv6 by taking the longest address.
-        List<InetAddress> hostAddresses = Arrays.asList(InetAddress.getAllByName(com.yahoo.net.HostName.getLocalhost()));
+        final List<InetAddress> hostAddresses;
+        try {
+            hostAddresses = Arrays.asList(InetAddress.getAllByName(HostName.getLocalhost()));
+        } catch (UnknownHostException e) {
+            throw new RuntimeException("Could not get hostAddresses", e);
+        }
+
         InetAddress hostAddress = Collections.max(hostAddresses,
                 (o1, o2) -> o1.getAddress().length - o2.getAddress().length);
 
-        NetworkAddressInterface networkAddressInterface = DockerNetworkCreator.getInterfaceForAddress(hostAddress);
+        final NetworkAddressInterface networkAddressInterface;
+        try {
+            networkAddressInterface = DockerNetworkCreator.getInterfaceForAddress(hostAddress);
+        } catch (SocketException|UnknownHostException e) {
+            throw new RuntimeException("Could not get getInterfaceForAddress for " + hostAddress, e);
+        }
         boolean isIPv6 = networkAddressInterface.interfaceAddress.getAddress() instanceof Inet6Address;
 
+        final String defaultGateway;
+        try {
+            defaultGateway = DockerNetworkCreator.getDefaultGatewayLinux(isIPv6).getHostAddress();
+        } catch (IOException | InterruptedException e) {
+            throw new RuntimeException("Could not find default gateway, isIpv6 is " + isIPv6, e);
+        }
+
         Network.Ipam ipam = new Network.Ipam().withConfig(new Network.Ipam.Config()
                 .withSubnet(hostAddress.getHostAddress() + "/" + networkAddressInterface.interfaceAddress.getNetworkPrefixLength())
-                .withGateway(DockerNetworkCreator.getDefaultGateway(isIPv6).getHostAddress()));
+                .withGateway(defaultGateway));
+
+        if (subnetwork.isPresent()) {
+            ipam = new Network.Ipam().withConfig(new Network.Ipam.Config().withSubnet(subnetwork.get()));
+        }
 
         Map<String, String> dockerNetworkOptions = new HashMap<>();
         dockerNetworkOptions.put("parent", networkAddressInterface.networkInterface.getDisplayName());
         dockerNetworkOptions.put("macvlan_mode", "bridge");
 
-        dockerClient.createNetworkCmd()
-                .withName(DOCKER_CUSTOM_MACVLAN_NETWORK_NAME)
-                .withDriver("macvlan")
-                .withEnableIpv6(isIPv6)
+        CreateNetworkCmd createNetworkCmd = dockerClient.createNetworkCmd()
                 .withIpam(ipam)
-                .withOptions(dockerNetworkOptions)
-                .exec();
+                .withName(DOCKER_CUSTOM_MACVLAN_NETWORK_NAME);
+        if (! subnetwork.isPresent()) {
+            createNetworkCmd
+                    .withDriver("macvlan")
+                    .withEnableIpv6(isIPv6)
+                    .withOptions(dockerNetworkOptions);
+        }
+        createNetworkCmd.exec();
     }
 
     @Override
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerNetworkCreator.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerNetworkCreator.java
index 32982d1c16c..03f8375e02d 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerNetworkCreator.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerNetworkCreator.java
@@ -14,7 +14,7 @@ import java.util.Collections;
  * @author valerijf
  */
 class DockerNetworkCreator {
-    static InetAddress getDefaultGateway(boolean ipv6) throws IOException, InterruptedException {
+    static InetAddress getDefaultGatewayLinux(boolean ipv6) throws IOException, InterruptedException {
         String command = ipv6 ? "route -A inet6 -n | grep 'UG[ \t]' | awk '{print $2}'" :
                 "route -n | grep 'UG[ \t]' | awk '{print $2}'";
 
diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
index a77776f53dc..e41074d9354 100644
--- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
+++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
@@ -3,15 +3,22 @@ package com.yahoo.vespa.hosted.dockerapi;
 
 import com.github.dockerjava.api.model.Network;
 import com.github.dockerjava.core.command.BuildImageResultCallback;
+import com.yahoo.metrics.simple.MetricReceiver;
+import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
+import org.apache.commons.io.IOUtils;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.Inet6Address;
 import java.net.InetAddress;
+import java.net.URL;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
+import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 
 import static junit.framework.TestCase.fail;
@@ -25,13 +32,23 @@ import static org.junit.Assume.assumeTrue;
  * Class for testing full integration with docker daemon, requires running daemon. To run these tests:
  *
  * MAC:
- *   1. Install Docker Toolbox, and start it (Docker Quick Start Terminal) (you can close terminal window afterwards)
+ *   1. Install Docker Toolbox, and start it (Docker Quickstart Terminal) (you can close terminal window afterwards)
+ *   2. For network test, we need to make docker containers visible for Mac: sudo route add 172.0.0.0/8 192.168.99.100
  *   2. Run tests from IDE/mvn.
  *
+
+ * TIPS:
+ *   For cleaning up your local docker machine (DON'T DO THIS ON PROD)
+ *     docker stop $(docker ps -a -q)
+ *     docker rm $(docker ps -a -q)
+ *
  * @author valerijf
  * @author dybdahl
  */
 public class DockerTest {
+    public static final String SUBNET_CONTAINER = "172.18.7.2/24";
+    public static final String IP_ADDRESS_FOR_CONTAINER = "172.18.7.32";
+
     private DockerImpl docker;
     private static final OS operatingSystem = getSystemOS();
     private static final String prefix = "/Users/" + System.getProperty("user.name") + "/.docker/machine/machines/default/";
@@ -51,10 +68,11 @@ public class DockerTest {
         assertThat(containers.isEmpty(), is(true));
     }
 
+    // It is ignored since it is a bit slow and unstable, at least on Mac.
+    @Ignore
     @Test
     public void testDockerImagePull() throws ExecutionException, InterruptedException {
         assumeTrue(dockerDaemonIsPresent());
-        DockerImpl docker = new DockerImpl(dockerConfig);
 
         docker.getAllManagedContainers();
         DockerImage dockerImage = new DockerImage("busybox:1.24.0");
@@ -74,33 +92,76 @@ public class DockerTest {
     }
 
     @Test
-    public void testCreateDeleteImageCreateDeleteContainer() throws IOException, InterruptedException, ExecutionException {
+    public void testContainerCycle() throws IOException, InterruptedException, ExecutionException {
         assumeTrue(dockerDaemonIsPresent());
-        createDockerImage(docker);
 
-        ContainerName containerName = new ContainerName("foo");
-        docker.stopContainer(containerName);
-        docker.deleteContainer(containerName);
-        assertThat(docker.getAllManagedContainers().isEmpty(), is(true));
-        docker.createContainerCommand(dockerImage, containerName, "hostName1").create();
-        List<Container> containers = docker.getAllManagedContainers();
-        assertThat(containers.size(), is(1));
-        docker.deleteContainer(containerName);
+        DockerImage busyBoxImage = new DockerImage("busybox:1.24.0");
+        // Pull the image and wait for the pull to complete
+        System.out.print("Pulling image.");
+        docker.pullImageAsync(busyBoxImage).get();
+        System.out.println("... done");
 
-        docker.pullImageAsync(dockerImage).get();
+        ContainerName containerName = new ContainerName("DummyWebServer");
 
-        // Translate the human readable ID to sha256-hash ID that is returned by getUnusedDockerImages()
-        DockerImage targetImage = new DockerImage(docker.dockerClient.inspectImageCmd(dockerImage.asString()).exec().getId());
-        Set<DockerImage> except = new HashSet<>();
-        List<DockerImage> x = docker.getUnusedDockerImages(except);
+        try { // Stop any old container
+            docker.stopContainer(containerName);
+            System.out.println("Stopped an old container with same name");
+        } catch (Exception e) {
+            System.out.println("Did not manage to stop container " + e.getMessage());
+        }
+        try { // Delete any remaining container
+            docker.deleteContainer(containerName);
+            System.out.println("Deleted an old container with same name");
+        } catch (Exception e) {
+            System.out.println("Did not manage to delete container " + e.getMessage());
+        }
 
-        // Remove the image
-        docker.deleteImage(dockerImage);
-        List<DockerImage> y = docker.getUnusedDockerImages(except);
+        int webPortForContainer = 6342;
 
-        assertFalse("Failed to delete " + dockerImage.asString() + " image", docker.imageIsDownloaded(dockerImage));
+        InetAddress inetAddress1 = Inet6Address.getByName(IP_ADDRESS_FOR_CONTAINER);
+        docker.createContainerCommand(busyBoxImage, containerName, "hostName1")
+                .withNetworkMode("vespa-macvlan")
+                .withIpAddress(inetAddress1)
+                .withCmd("sleep").withCmd("55555").create();
+
+        System.out.print("Starting container with sleep running.");
+        docker.startContainer(containerName);
+        System.out.println("..done");
+
+
+        // TODO: Get the container to start with httpd instead of hacking it here.
+        new Thread(() -> {
+            try {
+                // This call is blocking
+                docker.executeInContainer(containerName, "/bin/sh", "-c", "/bin/httpd -f -p " + webPortForContainer).getOutput();
+            } catch (RuntimeException e) {
+                // ignore, fails on shut down
+            }
+        }).start();
+
+        boolean success = false;
+        for (int x = 0; x < 600; x++) {
+            try {
+                URL url = new URL("http://" + IP_ADDRESS_FOR_CONTAINER + ":" + webPortForContainer);
+                InputStream is = url.openStream();
+                String containerServer = IOUtils.toString(is);
+                assertThat(containerServer, is("sdf"));
+            } catch (FileNotFoundException e) {
+                System.out.println("Managed to talk to web server in container.");
+                success = true;
+                break;
+            } catch (Throwable t) {
+                System.err.println(t.getMessage());
+                Thread.sleep(100);
+            }
+        }
+        docker.stopContainer(containerName);
+        docker.deleteContainer(containerName);
+        // Do not bother, it takes forever to load it again.. docker.deleteImage(busyBoxImage);
+        assertThat(success, is(true));
     }
 
+
     @Test
     public void testDockerNetworking() throws InterruptedException, ExecutionException, IOException {
         assumeTrue(dockerDaemonIsPresent());
@@ -142,14 +203,6 @@ public class DockerTest {
         }
     }
 
-    private void createDockerTestNetworkIfNeeded() {
-        if (! docker.dockerClient.listNetworksCmd().withNameFilter(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).exec().isEmpty()) return;
-
-        Network.Ipam ipam = new Network.Ipam().withConfig(new Network.Ipam.Config().withSubnet("172.18.0.0/16"));
-        docker.dockerClient.createNetworkCmd()
-                .withName(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withDriver("bridge").withIpam(ipam).exec();
-    }
-
     private boolean dockerDaemonIsPresent() {
         if (docker != null) return true;
         if (operatingSystem == OS.Unsupported) {
@@ -158,8 +211,7 @@ public class DockerTest {
         }
 
         try {
-            docker = new DockerImpl(dockerConfig);
-            createDockerTestNetworkIfNeeded();
+            setDocker();
             return true;
         } catch (Exception e) {
             System.out.println("Please install Docker Toolbox and start Docker Quick Start Terminal once, ignoring test.");
@@ -168,6 +220,17 @@ public class DockerTest {
         }
     }
 
+    private void setDocker() {
+        docker = new DockerImpl(
+                dockerConfig,
+                false, /* fallback to 1.23 on errors */
+                false, /* try setup netowork */
+                operatingSystem == OS.Mac_OS_X ? Optional.of(SUBNET_CONTAINER) : Optional.empty(), /* subnetwork */
+                false, /* hackAroundPullImageDueToJerseyConflicts */
+                100 /* dockerConnectTimeoutMillis */,
+                new MetricReceiverWrapper(MetricReceiver.nullImplementation));
+    }
+
     private void testReachabilityFromHost(ContainerName containerName, InetAddress target) throws IOException, InterruptedException {
         String[] curlNodeFromHost = {"curl", "-g", "http://" + target.getHostAddress() + "/ping"};
         while (!exec(curlNodeFromHost).equals("pong\n")) {