Add tests for mac.

2 files changed, 193 insertions, 61 deletions

diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
index 3d2b7c590e7..b82759aeb1b 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
@@ -21,6 +21,7 @@ import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory;
 import com.google.inject.Inject;
 import com.yahoo.collections.Pair;
 import com.yahoo.log.LogLevel;
+import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.net.HostName;
 import com.yahoo.system.ProcessExecuter;
 import com.yahoo.vespa.defaults.Defaults;
@@ -75,11 +76,32 @@ public class DockerImpl implements Docker {
 
     private GaugeWrapper numberOfRunningContainersGauge;
     private CounterWrapper numberOfDockerDaemonFails;
+    private final boolean hackAroundPullImageDueToJerseyConflicts;
 
+    // For testing
     DockerImpl(final DockerClient dockerClient) {
+        hackAroundPullImageDueToJerseyConflicts = false;
         this.dockerClient = dockerClient;
     }
 
+    // For testing
+    public DockerImpl(final DockerConfig config) {
+        hackAroundPullImageDueToJerseyConflicts = false;
+        JerseyDockerCmdExecFactory dockerFactory = new JerseyDockerCmdExecFactory()
+                .withMaxPerRouteConnections(DOCKER_MAX_PER_ROUTE_CONNECTIONS)
+                .withMaxTotalConnections(DOCKER_MAX_TOTAL_CONNECTIONS)
+                .withConnectTimeout(DOCKER_CONNECT_TIMEOUT_MILLIS)
+                .withReadTimeout(DOCKER_READ_TIMEOUT_MILLIS);
+        // Fail fast
+        RemoteApiVersion remoteApiVersion = findCorrectRemoteApiVersion(config, 100 /* connect timeout millis */);
+        this.dockerClient = DockerClientImpl.getInstance(
+                buildDockerClientConfig(config)
+                        .withApiVersion(remoteApiVersion)
+                        .build())
+                .withDockerCmdExecFactory(dockerFactory);
+        setMetrics(new MetricReceiverWrapper(MetricReceiver.nullImplementation));
+    }
+
     @Inject
     public DockerImpl(final DockerConfig config, MetricReceiverWrapper metricReceiver) {
         JerseyDockerCmdExecFactory dockerFactory =  new JerseyDockerCmdExecFactory()
@@ -87,12 +109,10 @@ public class DockerImpl implements Docker {
                 .withMaxTotalConnections(DOCKER_MAX_TOTAL_CONNECTIONS)
                 .withConnectTimeout(DOCKER_CONNECT_TIMEOUT_MILLIS)
                 .withReadTimeout(DOCKER_READ_TIMEOUT_MILLIS);
-
+        hackAroundPullImageDueToJerseyConflicts = true;
         RemoteApiVersion remoteApiVersion;
         try {
-            remoteApiVersion = RemoteApiVersion.parseConfig(DockerClientImpl.getInstance(
-                    buildDockerClientConfig(config).build())
-                    .withDockerCmdExecFactory(dockerFactory).versionCmd().exec().getApiVersion());
+            remoteApiVersion = getRemoteApiVersion(config, DOCKER_CONNECT_TIMEOUT_MILLIS);
             logger.info("Found version of remote docker API: "+ remoteApiVersion);
             // From version 1.24 a field was removed which causes trouble with the current docker java code.
             // When this is fixed, we can remove this and do not specify version.
@@ -116,16 +136,7 @@ public class DockerImpl implements Docker {
         } catch (Exception e) {
             throw new RuntimeException("Could not setup docker network", e);
         }
-
-        Dimensions dimensions = new Dimensions.Builder()
-                .add("host", HostName.getLocalhost())
-                .add("role", "docker").build();
-
-        numberOfRunningContainersGauge = metricReceiver.declareGauge(dimensions, "containers.running");
-        numberOfDockerDaemonFails = metricReceiver.declareCounter(dimensions, "daemon.api_fails");
-
-        // Some containers could already be running, count them and intialize to that value
-        numberOfRunningContainersGauge.sample(getAllManagedContainers().size());
+        setMetrics(metricReceiver);
     }
 
     static DefaultDockerClientConfig.Builder buildDockerClientConfig(DockerConfig config) {
@@ -188,10 +199,14 @@ public class DockerImpl implements Docker {
             completionListener = new CompletableFuture<>();
             scheduledPulls.put(image, completionListener);
         }
-        //dockerClient.pullImageCmd(image.asString()).exec(new ImagePullCallback(image));
-        // TODO: Need to call out to a command-line tool due to conflicting jackson dependencies between
-        // docker-java and pre-installed bundles in jdisc container
-        pullImageWithCommandTool(image, new ImagePullCallback(image), completionListener);
+        if (hackAroundPullImageDueToJerseyConflicts) {
+            // TODO: Need to call out to a command-line tool due to conflicting jackson dependencies between
+            // docker-java and pre-installed bundles in jdisc container
+            pullImageWithCommandTool(image, new ImagePullCallback(image), completionListener);
+        } else {
+            dockerClient.pullImageCmd(image.asString()).exec(new ImagePullCallback(image));
+        }
+
         return completionListener;
     }
 
@@ -547,4 +562,46 @@ public class DockerImpl implements Docker {
             }
         }
     }
+
+    private RemoteApiVersion findCorrectRemoteApiVersion(final DockerConfig config, int connectTimeousMillis) {
+        RemoteApiVersion remoteApiVersion;
+        try {
+            // Fail fast
+            remoteApiVersion = getRemoteApiVersion(config, connectTimeousMillis);
+            logger.info("Found version of remote docker API: " + remoteApiVersion);
+            // From version 1.24 a field was removed which causes trouble with the current docker java code.
+            // When this is fixed, we can remove this and do not specify version.
+            if (remoteApiVersion.isGreaterOrEqual(RemoteApiVersion.VERSION_1_24)) {
+                logger.info("Found version 1.24 or newer of remote API, using 1.23.");
+                return RemoteApiVersion.VERSION_1_23;
+            }
+            return remoteApiVersion;
+        } catch (Exception e) {
+            logger.log(LogLevel.ERROR, "Failed when trying to figure out remote API version of docker, using 1.23", e);
+            return RemoteApiVersion.VERSION_1_23;
+        }
+    }
+
+    private static RemoteApiVersion getRemoteApiVersion(final DockerConfig config, int connectTimeousMs) {
+        JerseyDockerCmdExecFactory dockerFactory = new JerseyDockerCmdExecFactory()
+                .withMaxPerRouteConnections(DOCKER_MAX_PER_ROUTE_CONNECTIONS)
+                .withMaxTotalConnections(DOCKER_MAX_TOTAL_CONNECTIONS)
+                .withConnectTimeout(connectTimeousMs)
+                .withReadTimeout(DOCKER_READ_TIMEOUT_MILLIS);
+        return RemoteApiVersion.parseConfig(DockerClientImpl.getInstance(
+                buildDockerClientConfig(config).build())
+                .withDockerCmdExecFactory(dockerFactory).versionCmd().exec().getApiVersion());
+    }
+
+    private void setMetrics(MetricReceiverWrapper metricReceiver) {
+        Dimensions dimensions = new Dimensions.Builder()
+                .add("host", HostName.getLocalhost())
+                .add("role", "docker").build();
+
+        numberOfRunningContainersGauge = metricReceiver.declareGauge(dimensions, "containers.running");
+        numberOfDockerDaemonFails = metricReceiver.declareCounter(dimensions, "daemon.api_fails");
+
+        // Some containers could already be running, count them and intialize to that value
+        numberOfRunningContainersGauge.sample(getAllManagedContainers().size());
+    }
 }
diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
index 6c5d6b1f3bb..964aa075cad 100644
--- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
+++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
@@ -5,8 +5,6 @@ import com.github.dockerjava.api.model.Network;
 import com.github.dockerjava.core.command.BuildImageResultCallback;
 import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
-import org.junit.After;
-import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -18,39 +16,65 @@ import java.net.InetAddress;
 import java.net.NetworkInterface;
 import java.net.SocketException;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
+import static junit.framework.TestCase.fail;
+import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
 
 /**
+ * Class for testing full integration with docker daemon, requires running daemon. To run these tests:
+ *
+ * MAC:
+ *   1. Install Docker Toolbox, and start it (Docker Quick Start Terminal) (you can close terminal window afterwards)
+ *   3. Run tests from IDE/mvn.
+ *
+ * LINUX:
+ *  1. Remove Ignore annotations
+ *  2. Change ownership of docker.sock
+ *      $ sudo chown <your username> /var/run/docker.sock
+ *  3. (Temporary) Manually create the docker network used by DockerImpl by running:
+ *      $ sudo docker network create --ipv6 --gateway=<your local IPv6 address> --subnet=fe80::1/16 habla
+ *  4. (Temporary) Manually build docker test image. Inside src/test/resources/simple-ipv6-server run:
+ *      $ sudo docker build -t "simple-ipv6-server:Dockerfile" .
+ *  5. (Temporary) Comment out createDockerImage() and shutdown()
+ *
  * @author valerijf
+ * @author dybdahl
  */
 public class DockerTest {
-    /**
-     * To run these tests:
-     *  1. Remove Ignore annotations
-     *  2. Change ownership of docker.sock
-     *      $ sudo chown <your username> /var/run/docker.sock
-     *  3. (Temporary) Manually create the docker network used by DockerImpl by running:
-     *      $ sudo docker network create --ipv6 --gateway=<your local IPv6 address> --subnet=fe80::1/16 habla
-     *  4. (Temporary) Manually build docker test image. Inside src/test/resources/simple-ipv6-server run:
-     *      $ sudo docker build -t "simple-ipv6-server:Dockerfile" .
-     *  5. (Temporary) Comment out setup() and shutdown()
-     */
+    private DockerImpl docker;
+    private static final boolean isMacOSX = System.getProperty("os.name").equals("Mac OS X");
+    private static final String prefix = "/Users/" + System.getProperty("user.name") + "/.docker/machine/machines/default/";
     private static final DockerConfig dockerConfig = new DockerConfig(new DockerConfig.Builder()
-            .caCertPath("")     // Temporary setting it to empty as this field is required, in the future
-            .clientCertPath("") // DockerConfig should be rewritten and probably moved to docker-api module
-            .clientKeyPath("")
-            .uri("unix:///var/run/docker.sock"));
-
-    private static final DockerImpl docker = new DockerImpl(dockerConfig, new MetricReceiverWrapper(MetricReceiver.nullImplementation));
+            .caCertPath(isMacOSX ? prefix + "ca.pem" : "")
+            .clientCertPath(isMacOSX ? prefix + "cert.pem" : "")
+            .clientKeyPath(isMacOSX ? prefix + "key.pem" : "")
+            .uri(isMacOSX ? "tcp://192.168.99.100:2376" : "unix:///var/run/docker.sock"));
     private static final DockerImage dockerImage = new DockerImage("simple-ipv6-server:Dockerfile");
 
+    @Test
+    public void testGetAllManagedContainersNoContainersRunning() {
+        assumeTrue(isMacOSX);
+        assumeTrue(dockerDaemonIsPresent());
+
+        List<Container> containers = docker.getAllManagedContainers();
+        assertThat(containers.isEmpty(), is(true));
+    }
 
-    @Ignore
     @Test
     public void testDockerImagePull() throws ExecutionException, InterruptedException {
+        assumeTrue(isMacOSX);
+        assumeTrue(dockerDaemonIsPresent());
+        DockerImpl docker = new DockerImpl(dockerConfig);
+
+        docker.getAllManagedContainers();
         DockerImage dockerImage = new DockerImage("busybox:1.24.0");
 
         // Pull the image and wait for the pull to complete
@@ -58,10 +82,40 @@ public class DockerTest {
 
         // Translate the human readable ID to sha256-hash ID that is returned by getUnusedDockerImages()
         DockerImage targetImage = new DockerImage(docker.dockerClient.inspectImageCmd(dockerImage.asString()).exec().getId());
-//        assertTrue("Image: " + dockerImage + " should be unused", docker.getUnusedDockerImages().contains(targetImage));
+        List<DockerImage> unusedDockerImages = docker.getUnusedDockerImages(new HashSet<>());
+        if (! unusedDockerImages.contains(dockerImage)) {
+            fail("Did not find image as unused, here are all images; " + unusedDockerImages);
+        }
+        // Remove the image
+        docker.deleteImage(dockerImage);
+        assertFalse("Failed to delete " + dockerImage.asString() + " image", docker.imageIsDownloaded(dockerImage));
+    }
+
+    @Test
+    public void testCreateImageStartAndStopContainerDeleteImage() throws IOException, InterruptedException, ExecutionException {
+        assumeTrue(isMacOSX);
+        assumeTrue(dockerDaemonIsPresent());
+        createDockerImage(docker);
+        ContainerName containerName = new ContainerName("foo");
+        docker.stopContainer(containerName);
+        docker.deleteContainer(containerName);
+        assertThat(docker.getAllManagedContainers().isEmpty(), is(true));
+        docker.createContainerCommand(dockerImage, containerName, "hostName1").create();
+        List<Container> containers = docker.getAllManagedContainers();
+        assertThat(containers.size(), is(1));
+        docker.deleteContainer(containerName);
+
+        docker.pullImageAsync(dockerImage).get();
+
+        // Translate the human readable ID to sha256-hash ID that is returned by getUnusedDockerImages()
+        DockerImage targetImage = new DockerImage(docker.dockerClient.inspectImageCmd(dockerImage.asString()).exec().getId());
+        Set<DockerImage> except = new HashSet<>();
+        List<DockerImage> x = docker.getUnusedDockerImages(except);
 
         // Remove the image
         docker.deleteImage(dockerImage);
+        List<DockerImage> y = docker.getUnusedDockerImages(except);
+
         assertFalse("Failed to delete " + dockerImage.asString() + " image", docker.imageIsDownloaded(dockerImage));
     }
 
@@ -74,6 +128,7 @@ public class DockerTest {
         ContainerName containerName2 = new ContainerName("test-container-2");
         InetAddress inetAddress1 = Inet6Address.getByName("fe80::10");
         InetAddress inetAddress2 = Inet6Address.getByName("fe80::11");
+        DockerImpl docker = new DockerImpl(dockerConfig, new MetricReceiverWrapper(MetricReceiver.nullImplementation));
 
          docker.createContainerCommand(dockerImage, containerName1, hostName1).withIpAddress(inetAddress1).create();
          docker.createContainerCommand(dockerImage, containerName2, hostName2).withIpAddress(inetAddress2).create();
@@ -98,6 +153,20 @@ public class DockerTest {
         }
     }
 
+    private boolean dockerDaemonIsPresent() {
+        if (!isMacOSX) {
+            System.out.println("This test does not support " + System.getProperty("os.name") + " yet, ignoring test.");
+            return false;
+        }
+        try {
+            docker = new DockerImpl(dockerConfig, new MetricReceiverWrapper(MetricReceiver.nullImplementation));
+            return true;
+        } catch (Exception e) {
+            System.out.println("Please install Docker Toolbox and start Docker Quick Start Terminal once, ignoring test.");
+            return false;
+        }
+    }
+
     private void testReachabilityFromHost(ContainerName containerName, InetAddress target) throws IOException, InterruptedException {
         String[] curlNodeFromHost = {"curl", "-g", "http://[" + target.getHostAddress() + "%" + getInterfaceName() + "]/ping"};
         while (!exec(curlNodeFromHost).equals("pong\n")) {
@@ -108,18 +177,6 @@ public class DockerTest {
 
 
     /**
-     * Returns IPv6 address of on the "docker0" interface that can be reached by the containers
-     */
-    private static String getLocalIPv6Address() throws SocketException {
-        return Collections.list(NetworkInterface.getNetworkInterfaces()).stream()
-                .filter(networkInterface -> networkInterface.getDisplayName().equals("docker0"))
-                .flatMap(i -> Collections.list(i.getInetAddresses()).stream())
-                .filter(ip -> ip instanceof Inet6Address && ip.isLinkLocalAddress())
-                .findFirst().orElseThrow(RuntimeException::new)
-                .getHostAddress().split("%")[0];
-    }
-
-    /**
      * Returns the display name of the bridge used by our custom docker network. This is used for routing in the
      * network tests. The bridge is assumed to be the only IPv6 interface starting with "br-"
      */
@@ -153,15 +210,19 @@ public class DockerTest {
         return ret.toString();
     }
 
-    @Before
-    public void setup() throws IOException, ExecutionException, InterruptedException {
-        // Build the image locally
-        File dockerFilePath = new File("src/test/resources/simple-ipv6-server");
-        docker.dockerClient
-                .buildImageCmd(dockerFilePath)
-                .withTag(dockerImage.asString()).exec(new BuildImageResultCallback()).awaitCompletion();
+    /**
+     * Returns IPv6 address of on the "docker0" interface that can be reached by the containers
+     */
+    private static String getLocalIPv6Address() throws SocketException {
+        return Collections.list(NetworkInterface.getNetworkInterfaces()).stream()
+                .filter(networkInterface -> networkInterface.getDisplayName().equals("docker0"))
+                .flatMap(i -> Collections.list(i.getInetAddresses()).stream())
+                .filter(ip -> ip instanceof Inet6Address && ip.isLinkLocalAddress())
+                .findFirst().orElseThrow(RuntimeException::new)
+                .getHostAddress().split("%")[0];
+    }
 
-        // Create a temporary network
+    private void createSomeNetork() throws SocketException {
         Network.Ipam ipam = new Network.Ipam().withConfig(new Network.Ipam.Config()
                 .withSubnet("fe80::1/16").withGateway(getLocalIPv6Address()));
         // TODO: This needs to match the network name in DockerOperations!?
@@ -169,10 +230,24 @@ public class DockerTest {
                 .withIpam(ipam).exec();
     }
 
-    @After
-    public void shutdown() {
+    private void remoteNetwork() {
         // Remove the network we created earlier
         // TODO: This needs to match the network name in DockerOperations!?
         docker.dockerClient.removeNetworkCmd("habla").exec();
     }
+
+    // TODO: Do we need this? Rather use network created by DockerImpl, right?
+    void createDockerImage(DockerImpl docker) throws IOException, ExecutionException, InterruptedException {
+        try {
+            docker.deleteImage(new DockerImage(dockerImage.asString()));
+        } catch (Exception e) {
+            assertThat(e.getMessage(), is("Failed to delete docker image simple-ipv6-server:Dockerfile"));
+        }
+
+        // Build the image locally
+        File dockerFilePath = new File("src/test/resources/simple-ipv6-server");
+        docker.dockerClient
+                .buildImageCmd(dockerFilePath)
+                .withTag(dockerImage.asString()).exec(new BuildImageResultCallback()).awaitCompletion();
+    }
 }