summaryrefslogtreecommitdiffstats
path: root/docker-api
diff options
context:
space:
mode:
authorValerij Fredriksen <valerijf@oath.com>2018-02-22 16:53:29 +0100
committerValerij Fredriksen <valerij92@gmail.com>2018-02-22 17:53:42 +0100
commit30505ca4b061767e72e4ba783d8d339b60bcdfbd (patch)
tree0489f4e9e00ba9e59ed30e5d0f31641ece80f4f5 /docker-api
parent7358ce6a49eedf51b3761a855921c58a5f813321 (diff)
Simplify DockerImpl
Diffstat (limited to 'docker-api')
-rw-r--r--docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java120
-rw-r--r--docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java94
-rw-r--r--docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/VespaSSLConfig.java228
-rw-r--r--docker-api/src/main/resources/configdefinitions/docker.def3
-rw-r--r--docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java61
-rw-r--r--docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java197
-rw-r--r--docker-api/src/test/resources/simple-ipv6-server/Dockerfile13
-rw-r--r--docker-api/src/test/resources/simple-ipv6-server/README10
-rw-r--r--docker-api/src/test/resources/simple-ipv6-server/src/fillmem.py11
-rw-r--r--docker-api/src/test/resources/simple-ipv6-server/src/server.py43
10 files changed, 30 insertions, 750 deletions
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
index e81c6325922..2da18e12e40 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
@@ -15,15 +15,14 @@ import com.github.dockerjava.api.model.Image;
import com.github.dockerjava.api.model.Network;
import com.github.dockerjava.api.model.Statistics;
import com.github.dockerjava.core.DefaultDockerClientConfig;
+import com.github.dockerjava.core.DockerClientConfig;
import com.github.dockerjava.core.DockerClientImpl;
-import com.github.dockerjava.core.RemoteApiVersion;
import com.github.dockerjava.core.async.ResultCallbackTemplate;
import com.github.dockerjava.core.command.BuildImageResultCallback;
import com.github.dockerjava.core.command.ExecStartResultCallback;
import com.github.dockerjava.core.command.PullImageResultCallback;
import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory;
import com.google.inject.Inject;
-import com.yahoo.log.LogLevel;
import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper;
import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions;
import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
@@ -34,7 +33,6 @@ import java.io.File;
import java.io.IOException;
import java.net.Inet6Address;
import java.net.InetAddress;
-import java.net.URI;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
@@ -58,13 +56,11 @@ public class DockerImpl implements Docker {
public static final String DOCKER_CUSTOM_MACVLAN_NETWORK_NAME = "vespa-macvlan";
static final String LABEL_NAME_MANAGEDBY = "com.yahoo.vespa.managedby";
-
- private final int SECONDS_TO_WAIT_BEFORE_KILLING;
- private final boolean fallbackTo123OnErrors;
private static final String FRAMEWORK_CONTAINER_PREFIX = "/";
+
private final DockerConfig config;
- private final boolean inProduction;
- private Optional<DockerImageGarbageCollector> dockerImageGC = Optional.empty();
+ private final Optional<DockerImageGarbageCollector> dockerImageGC;
+ private final int secondsToWaitBeforeKilling;
private CounterWrapper numberOfDockerDaemonFails;
private boolean started = false;
@@ -76,63 +72,40 @@ public class DockerImpl implements Docker {
DockerClient dockerClient;
@Inject
- public DockerImpl(final DockerConfig config, MetricReceiverWrapper metricReceiver) {
- this(config,
- true, /* fallback to 1.23 on errors */
- metricReceiver,
- !config.isRunningLocally());
- }
-
- private DockerImpl(final DockerConfig config,
- boolean fallbackTo123OnErrors,
- MetricReceiverWrapper metricReceiverWrapper,
- boolean inProduction) {
+ public DockerImpl(DockerConfig config, MetricReceiverWrapper metricReceiverWrapper) {
this.config = config;
- this.fallbackTo123OnErrors = fallbackTo123OnErrors;
- this.inProduction = inProduction;
- if (config == null) {
- this.SECONDS_TO_WAIT_BEFORE_KILLING = 10;
- } else {
- SECONDS_TO_WAIT_BEFORE_KILLING = config.secondsToWaitBeforeKillingContainer();
- }
- if (metricReceiverWrapper != null) {
- setMetrics(metricReceiverWrapper);
- }
+
+ secondsToWaitBeforeKilling = Optional.ofNullable(config)
+ .map(DockerConfig::secondsToWaitBeforeKillingContainer)
+ .orElse(10);
+
+ dockerImageGC = Optional.ofNullable(config)
+ .map(DockerConfig::imageGCMinTimeToLiveMinutes)
+ .map(Duration::ofMinutes)
+ .map(DockerImageGarbageCollector::new);
+
+ Optional.ofNullable(metricReceiverWrapper).ifPresent(this::setMetrics);
}
// For testing
DockerImpl(final DockerClient dockerClient) {
- this(null, false, null, false);
+ this(null, null);
this.dockerClient = dockerClient;
}
- // For testing
- DockerImpl(final DockerConfig config,
- boolean fallbackTo123OnErrors,
- MetricReceiverWrapper metricReceiverWrapper) {
- this(config, fallbackTo123OnErrors, metricReceiverWrapper, false);
- }
-
@Override
public void start() {
if (started) return;
started = true;
if (config != null) {
- if (dockerClient == null) {
- dockerClient = initDockerConnection();
- }
- if (inProduction) {
- Duration minAgeToDelete = Duration.ofMinutes(config.imageGCMinTimeToLiveMinutes());
- dockerImageGC = Optional.of(new DockerImageGarbageCollector(minAgeToDelete));
-
+ dockerClient = createDockerClient(config);
- if (!config.networkNATed()) {
- try {
- setupDockerNetworkIfNeeded();
- } catch (Exception e) {
- throw new DockerException("Could not setup docker network", e);
- }
+ if (!config.networkNATed()) {
+ try {
+ setupDockerNetworkIfNeeded();
+ } catch (Exception e) {
+ throw new DockerException("Could not setup docker network", e);
}
}
}
@@ -143,21 +116,6 @@ public class DockerImpl implements Docker {
return config.networkNATed();
}
- static DefaultDockerClientConfig.Builder buildDockerClientConfig(DockerConfig config) {
- DefaultDockerClientConfig.Builder dockerConfigBuilder = new DefaultDockerClientConfig.Builder()
- .withDockerHost(config.uri());
-
- if (URI.create(config.uri()).getScheme().equals("tcp") && !config.caCertPath().isEmpty()) {
- // In current version of docker-java (3.0.2), withDockerTlsVerify() only effect is when using it together
- // with withDockerCertPath(), where setting withDockerTlsVerify() must be set to true, otherwise the
- // cert path parameter will be ignored.
- // withDockerTlsVerify() has no effect when used with withCustomSslConfig()
- dockerConfigBuilder.withCustomSslConfig(new VespaSSLConfig(config));
- }
-
- return dockerConfigBuilder;
- }
-
private void setupDockerNetworkIfNeeded() throws IOException {
if (!dockerClient.listNetworksCmd().withNameFilter(DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).exec().isEmpty()) return;
@@ -366,7 +324,7 @@ public class DockerImpl implements Docker {
@Override
public void stopContainer(final ContainerName containerName) {
try {
- dockerClient.stopContainerCmd(containerName.asString()).withTimeout(SECONDS_TO_WAIT_BEFORE_KILLING).exec();
+ dockerClient.stopContainerCmd(containerName.asString()).withTimeout(secondsToWaitBeforeKilling).exec();
} catch (NotModifiedException ignored) {
// If is already stopped, ignore
} catch (RuntimeException e) {
@@ -545,36 +503,18 @@ public class DockerImpl implements Docker {
}
}
- private DockerClient initDockerConnection() {
+ private static DockerClient createDockerClient(DockerConfig config) {
JerseyDockerCmdExecFactory dockerFactory = new JerseyDockerCmdExecFactory()
.withMaxPerRouteConnections(config.maxPerRouteConnections())
.withMaxTotalConnections(config.maxTotalConnections())
.withConnectTimeout(config.connectTimeoutMillis())
.withReadTimeout(config.readTimeoutMillis());
- RemoteApiVersion remoteApiVersion;
- try {
- remoteApiVersion = RemoteApiVersion.parseConfig(DockerClientImpl.getInstance(
- buildDockerClientConfig(config).build())
- .withDockerCmdExecFactory(dockerFactory).versionCmd().exec().getApiVersion());
- logger.info("Found version of remote docker API: " + remoteApiVersion);
- // From version 1.24 a field was removed which causes trouble with the current docker java code.
- // When this is fixed, we can remove this and do not specify version.
- if (remoteApiVersion.isGreaterOrEqual(RemoteApiVersion.VERSION_1_24)) {
- remoteApiVersion = RemoteApiVersion.VERSION_1_23;
- logger.info("Found version 1.24 or newer of remote API, using 1.23.");
- }
- } catch (Exception e) {
- if (!fallbackTo123OnErrors) {
- throw e;
- }
- logger.log(LogLevel.ERROR, "Failed when trying to figure out remote API version of docker, using 1.23", e);
- remoteApiVersion = RemoteApiVersion.VERSION_1_23;
- }
- return DockerClientImpl.getInstance(
- buildDockerClientConfig(config)
- .withApiVersion(remoteApiVersion)
- .build())
+ DockerClientConfig dockerClientConfig = new DefaultDockerClientConfig.Builder()
+ .withDockerHost(config.uri())
+ .build();
+
+ return DockerClientImpl.getInstance(dockerClientConfig)
.withDockerCmdExecFactory(dockerFactory);
}
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java
deleted file mode 100644
index 549af0d85cb..00000000000
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java
+++ /dev/null
@@ -1,94 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.dockerapi;
-
-import com.github.dockerjava.api.model.Network;
-import com.yahoo.metrics.simple.MetricReceiver;
-import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
-
-import java.io.File;
-
-/**
- * Helper class for testing full integration with docker daemon, requires running daemon. To run these tests:
- *
- * MAC:
- * 1. Install Docker Toolbox, and start it (Docker Quickstart Terminal) (you can close terminal window afterwards)
- * 2. For network test, we need to make docker containers visible for Mac: sudo route add 172.18.0.0/16 192.168.99.100
- *
- * @author freva
- */
-public class DockerTestUtils {
- private static final OS operatingSystem = getSystemOS();
- private static final String prefix = "/Users/" + System.getProperty("user.name") + "/.docker/machine/machines/default/";
- private static final DockerConfig dockerConfig = new DockerConfig(new DockerConfig.Builder()
- .caCertPath( operatingSystem == OS.Mac_OS_X ? prefix + "ca.pem" : "")
- .clientCertPath(operatingSystem == OS.Mac_OS_X ? prefix + "cert.pem" : "")
- .clientKeyPath( operatingSystem == OS.Mac_OS_X ? prefix + "key.pem" : "")
- .uri( operatingSystem == OS.Mac_OS_X ? "tcp://192.168.99.100:2376" : "tcp://localhost:2376")
- .secondsToWaitBeforeKillingContainer(0));
- private static DockerImpl docker;
-
- public static boolean dockerDaemonIsPresent() {
- if (docker != null) return true;
- if (operatingSystem == OS.Unsupported) {
- System.err.println("This test does not support " + System.getProperty("os.name") + " yet, ignoring test.");
- return false;
- }
-
- try {
- getDocker(); // Will throw an exception if docker is not installed/incorrectly configured
- return true;
- } catch (Exception e) {
- System.err.println("Please install Docker Toolbox and start Docker Quick Start Terminal once, ignoring test.");
- System.err.println(e.getMessage());
- return false;
- }
- }
-
- public static DockerImpl getDocker() {
- if (docker == null) {
- DockerImpl tmpDocker = new DockerImpl(
- dockerConfig,
- false, /* fallback to 1.23 on errors */
- new MetricReceiverWrapper(MetricReceiver.nullImplementation));
- tmpDocker.start();
- createDockerTestNetworkIfNeeded(tmpDocker);
- docker = tmpDocker;
- }
-
- return docker;
- }
-
- public static void createDockerTestNetworkIfNeeded(DockerImpl docker) {
- if (! docker.dockerClient.listNetworksCmd().withNameFilter(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).exec().isEmpty()) return;
-
- Network.Ipam ipam = new Network.Ipam().withConfig(new Network.Ipam.Config()
- .withSubnet("172.18.0.0/16")
- .withGateway("172.18.0.1"));
- docker.dockerClient.createNetworkCmd()
- .withName(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withDriver("bridge").withIpam(ipam).exec();
- }
-
- public static void buildSimpleHttpServerDockerImage(DockerImpl docker, DockerImage dockerImage) {
- try {
- docker.deleteImage(dockerImage);
- } catch (Exception e) {
- if (! e.getMessage().equals("Failed to delete docker image " + dockerImage.asString())) {
- throw e;
- }
- }
-
- // Build the image locally
- File dockerFileStream = new File("src/test/resources/simple-ipv6-server");
- docker.buildImage(dockerFileStream, dockerImage);
- }
-
- public enum OS { Linux, Mac_OS_X, Unsupported }
-
- public static OS getSystemOS() {
- switch (System.getProperty("os.name").toLowerCase()) {
- case "linux": return OS.Linux;
- case "mac os x": return OS.Mac_OS_X;
- default: return OS.Unsupported;
- }
- }
-}
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/VespaSSLConfig.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/VespaSSLConfig.java
deleted file mode 100644
index e9bc0181dd7..00000000000
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/VespaSSLConfig.java
+++ /dev/null
@@ -1,228 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.dockerapi;
-
-import com.github.dockerjava.api.exception.DockerClientException;
-import com.github.dockerjava.core.SSLConfig;
-import org.bouncycastle.asn1.ASN1ObjectIdentifier;
-import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
-import org.bouncycastle.cert.X509CertificateHolder;
-import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
-import org.bouncycastle.openssl.PEMKeyPair;
-import org.bouncycastle.openssl.PEMParser;
-import org.glassfish.jersey.SslConfigurator;
-
-import javax.net.ssl.SSLContext;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.security.KeyFactory;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.PrivateKey;
-import java.security.Security;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateException;
-import java.security.spec.InvalidKeySpecException;
-import java.security.spec.PKCS8EncodedKeySpec;
-import java.util.ArrayList;
-import java.util.List;
-
-import static java.util.Objects.requireNonNull;
-
-
-/**
- * This class is based off {@link com.github.dockerjava.core.LocalDirectorySSLConfig}, but with the ability to
- * specify path to each of the certificates instead of directory path. Additionally it includes
- * {@link com.github.dockerjava.core.util.CertificateUtils} because of version conflict of with
- * com.google.code.findbugs.annotations
- */
-public class VespaSSLConfig implements SSLConfig {
- private final DockerConfig config;
-
- public VespaSSLConfig(DockerConfig config) {
- this.config = config;
- }
-
- @Override
- public SSLContext getSSLContext() {
- try {
- Security.addProvider(new BouncyCastleProvider());
-
- // properties acrobatics not needed for java > 1.6
- String httpProtocols = System.getProperty("https.protocols");
- System.setProperty("https.protocols", "TLSv1");
- SslConfigurator sslConfig = SslConfigurator.newInstance(true);
- if (httpProtocols != null) {
- System.setProperty("https.protocols", httpProtocols);
- }
-
- String keypem = new String(Files.readAllBytes(Paths.get(config.clientKeyPath())));
- String certpem = new String(Files.readAllBytes(Paths.get(config.clientCertPath())));
- String capem = new String(Files.readAllBytes(Paths.get(config.caCertPath())));
-
- sslConfig.keyStore(createKeyStore(keypem, certpem));
- sslConfig.keyStorePassword("docker");
- sslConfig.trustStore(createTrustStore(capem));
-
- return sslConfig.createSSLContext();
- } catch (Exception e) {
- throw new DockerClientException(e.getMessage(), e);
- }
- }
-
- public static KeyStore createKeyStore(final String keypem, final String certpem) throws NoSuchAlgorithmException,
- IOException, CertificateException, KeyStoreException {
- PrivateKey privateKey = loadPrivateKey(keypem);
- requireNonNull(privateKey);
- List<Certificate> privateCertificates = loadCertificates(certpem);
-
- KeyStore keyStore = KeyStore.getInstance("JKS");
- keyStore.load(null);
-
- keyStore.setKeyEntry("docker",
- privateKey,
- "docker".toCharArray(),
- privateCertificates.toArray(new Certificate[privateCertificates.size()])
- );
-
- return keyStore;
- }
-
- /**
- * from "cert.pem" String
- */
- private static List<Certificate> loadCertificates(final String certpem) throws IOException,
- CertificateException {
- final StringReader certReader = new StringReader(certpem);
- try (BufferedReader reader = new BufferedReader(certReader)) {
- return loadCertificates(reader);
- }
- }
-
- /**
- * "cert.pem" from reader
- */
- private static List<Certificate> loadCertificates(final Reader reader) throws IOException,
- CertificateException {
- try (PEMParser pemParser = new PEMParser(reader)) {
- List<Certificate> certificates = new ArrayList<>();
-
- JcaX509CertificateConverter certificateConverter = new JcaX509CertificateConverter().setProvider("BC");
- Object certObj = pemParser.readObject();
-
- if (certObj instanceof X509CertificateHolder) {
- X509CertificateHolder certificateHolder = (X509CertificateHolder) certObj;
- certificates.add(certificateConverter.getCertificate(certificateHolder));
- }
-
- return certificates;
- }
- }
-
-
- /**
- * Return private key ("key.pem") from Reader
- */
- private static PrivateKey loadPrivateKey(final Reader reader) throws IOException, NoSuchAlgorithmException {
- try (PEMParser pemParser = new PEMParser(reader)) {
- Object readObject = pemParser.readObject();
- while (readObject != null) {
- if (readObject instanceof PEMKeyPair) {
- PEMKeyPair pemKeyPair = (PEMKeyPair) readObject;
- PrivateKey privateKey = guessKey(pemKeyPair.getPrivateKeyInfo().getEncoded());
- if (privateKey != null) {
- return privateKey;
- }
- } else if (readObject instanceof PrivateKeyInfo) {
- PrivateKeyInfo privateKeyInfo = (PrivateKeyInfo) readObject;
- PrivateKey privateKey = guessKey(privateKeyInfo.getEncoded());
- if (privateKey != null) {
- return privateKey;
- }
- } else if (readObject instanceof ASN1ObjectIdentifier) {
- // no idea how it can be used
- final ASN1ObjectIdentifier asn1ObjectIdentifier = (ASN1ObjectIdentifier) readObject;
- }
-
- readObject = pemParser.readObject();
- }
- }
-
- return null;
- }
-
- private static PrivateKey guessKey(byte[] encodedKey) throws NoSuchAlgorithmException {
- //no way to know, so iterate
- for (String guessFactory : new String[]{"RSA", "ECDSA"}) {
- try {
- KeyFactory factory = KeyFactory.getInstance(guessFactory);
-
- PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(encodedKey);
- return factory.generatePrivate(privateKeySpec);
- } catch (InvalidKeySpecException ignore) {
- }
- }
-
- return null;
- }
-
- /**
- * Return KeyPair from "key.pem"
- */
- private static PrivateKey loadPrivateKey(final String keypem) throws IOException, NoSuchAlgorithmException {
- try (StringReader certReader = new StringReader(keypem);
- BufferedReader reader = new BufferedReader(certReader)) {
- return loadPrivateKey(reader);
- }
- }
-
- /**
- * "ca.pem" from String
- */
- public static KeyStore createTrustStore(String capem) throws IOException, CertificateException,
- KeyStoreException, NoSuchAlgorithmException {
- try (Reader certReader = new StringReader(capem)) {
- return createTrustStore(certReader);
- }
- }
-
- /**
- * "ca.pem" from Reader
- */
- public static KeyStore createTrustStore(final Reader certReader) throws IOException, CertificateException,
- KeyStoreException, NoSuchAlgorithmException {
- try (PEMParser pemParser = new PEMParser(certReader)) {
- X509CertificateHolder certificateHolder = (X509CertificateHolder) pemParser.readObject();
- Certificate caCertificate = new JcaX509CertificateConverter()
- .setProvider("BC")
- .getCertificate(certificateHolder);
-
- KeyStore trustStore = KeyStore.getInstance("JKS");
- trustStore.load(null);
- trustStore.setCertificateEntry("ca", caCertificate);
-
- return trustStore;
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- VespaSSLConfig that = (VespaSSLConfig) o;
-
- return config.equals(that.config);
-
- }
-
- @Override
- public int hashCode() {
- return config.hashCode();
- }
-}
diff --git a/docker-api/src/main/resources/configdefinitions/docker.def b/docker-api/src/main/resources/configdefinitions/docker.def
index b4585318cd8..83fee05dff6 100644
--- a/docker-api/src/main/resources/configdefinitions/docker.def
+++ b/docker-api/src/main/resources/configdefinitions/docker.def
@@ -1,9 +1,6 @@
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
namespace=vespa.hosted.dockerapi
-caCertPath string default = ""
-clientCertPath string default = ""
-clientKeyPath string default = ""
uri string default = "unix:///host/var/run/docker.sock"
secondsToWaitBeforeKillingContainer int default = 10
diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java
index 12e52dde494..654b5df3f3b 100644
--- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java
+++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java
@@ -12,7 +12,6 @@ import com.github.dockerjava.api.command.InspectImageCmd;
import com.github.dockerjava.api.command.InspectImageResponse;
import com.github.dockerjava.api.command.PullImageCmd;
import com.github.dockerjava.api.exception.NotFoundException;
-import com.github.dockerjava.core.DefaultDockerClientConfig;
import com.github.dockerjava.core.command.ExecStartResultCallback;
import com.yahoo.metrics.simple.MetricReceiver;
import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
@@ -20,12 +19,6 @@ import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Matchers;
-import java.io.IOException;
-import java.security.KeyManagementException;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.UnrecoverableKeyException;
-
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
@@ -39,60 +32,6 @@ import static org.mockito.Mockito.when;
* @author tonytv
*/
public class DockerImplTest {
- @Test
- public void testDockerConfigWithUnixPath() throws UnrecoverableKeyException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
- String dockerUri = "unix:///var/run/docker.sock";
- DockerConfig config = createConfig(dockerUri, null, null, null);
- DefaultDockerClientConfig clientConfig = DockerImpl.buildDockerClientConfig(config).build();
-
- assertTrue("Docker uri incorrectly set", clientConfig.getDockerHost().toString().equals(dockerUri));
- assertTrue("SSL config was set when using socket", clientConfig.getSSLConfig() == null);
- }
-
- @Test
- public void testDockerConfigWithTcpPathWithoutSSL() {
- String dockerUri = "tcp://127.0.0.1:2376";
- DockerConfig config = createConfig(dockerUri, null, null, null);
- DefaultDockerClientConfig clientConfig = DockerImpl.buildDockerClientConfig(config).build();
-
- assertTrue("Docker uri incorrectly set", clientConfig.getDockerHost().toString().equals(dockerUri));
- assertTrue("SSL config was set", clientConfig.getSSLConfig() == null);
- }
-
- @Test
- public void testDockerConfigWithTcpPathWithSslConfig() throws IOException, UnrecoverableKeyException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
- String dockerUri = "tcp://127.0.0.1:2376";
- DockerConfig config = createConfig(dockerUri, "/some/path/ca", "/some/path/cert", "/some/path/key");
- DefaultDockerClientConfig clientConfig = DockerImpl.buildDockerClientConfig(config).build();
-
- assertTrue("Docker uri incorrectly set", clientConfig.getDockerHost().toString().equals(dockerUri));
- assertTrue("SSL config was not set", clientConfig.getSSLConfig() != null);
- }
-
- @Test(expected=RuntimeException.class)
- public void testDockerConfigWithTcpPathWithInvalidSslConfig() throws IOException, UnrecoverableKeyException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
- String dockerUri = "tcp://127.0.0.1:2376";
- DockerConfig config = createConfig(dockerUri, "/some/path/ca", "/some/path/cert", "/some/path/key");
- DefaultDockerClientConfig clientConfig = DockerImpl.buildDockerClientConfig(config).build();
-
- assertTrue("Docker uri incorrectly set", clientConfig.getDockerHost().toString().equals(dockerUri));
- assertTrue("SSL config was not set", clientConfig.getSSLConfig() != null);
-
- // SSL certificates are read during the getSSLContext(), the invalid paths should cause a RuntimeException
- clientConfig.getSSLConfig().getSSLContext();
- }
-
- private static DockerConfig createConfig(String uri, String caCertPath, String clientCertPath, String clientKeyPath) {
- DockerConfig.Builder configBuilder = new DockerConfig.Builder();
-
- if (uri != null) configBuilder.uri(uri);
- if (caCertPath != null) configBuilder.caCertPath(caCertPath);
- if (clientCertPath != null) configBuilder.clientCertPath(clientCertPath);
- if (clientKeyPath != null) configBuilder.clientKeyPath(clientKeyPath);
-
- return new DockerConfig(configBuilder);
- }
-
@Test
public void testExecuteCompletes() throws Exception {
diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
deleted file mode 100644
index 18f87e5ae17..00000000000
--- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java
+++ /dev/null
@@ -1,197 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.dockerapi;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URL;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
-
-/**
- * Requires docker daemon, see {@link com.yahoo.vespa.hosted.dockerapi.DockerTestUtils} for more details.
- *
- * @author freva
- * @author dybdahl
- */
-public class DockerTest {
- private DockerImpl docker;
- private static final DockerImage dockerImage = new DockerImage("simple-ipv6-server:Dockerfile");
- private static final String MANAGER_NAME = "docker-test";
-
- // Ignored because the test is very slow (several minutes) when swap is enabled, to disable: (Linux)
- // $ sudo swapoff -a
- @Ignore
- @Test
- public void testOutOfMemoryDoesNotAffectOtherContainers() throws InterruptedException, ExecutionException, IOException {
- String hostName1 = "docker10.test.yahoo.com";
- String hostName2 = "docker11.test.yahoo.com";
- ContainerName containerName1 = new ContainerName("docker-test-1");
- ContainerName containerName2 = new ContainerName("docker-test-2");
- InetAddress inetAddress1 = InetAddress.getByName("172.18.10.10");
- InetAddress inetAddress2 = InetAddress.getByName("172.18.10.11");
-
- docker.createContainerCommand(dockerImage, ContainerResources.from(0, 0.1), containerName1, hostName1)
- .withManagedBy(MANAGER_NAME)
- .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME)
- .withIpAddress(inetAddress1)
- .create();
- docker.startContainer(containerName1);
-
- docker.createContainerCommand(dockerImage, ContainerResources.from(0, 0.1), containerName2, hostName2)
- .withManagedBy(MANAGER_NAME)
- .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME)
- .withIpAddress(inetAddress2)
- .create();
- docker.startContainer(containerName2);
-
- // 137 = 128 + 9 = kill -9 (SIGKILL), doesn't need to be run as "root", but "yahoo" does not exist in this basic image
- assertThat(docker.executeInContainerAsRoot(containerName2, "python", "/pysrc/fillmem.py", "90").getExitStatus(), is(137));
-
- // Verify that both HTTP servers are still up
- testReachabilityFromHost("http://" + inetAddress1.getHostAddress() + "/ping");
- testReachabilityFromHost("http://" + inetAddress2.getHostAddress() + "/ping");
-
- docker.stopContainer(containerName1);
- docker.deleteContainer(containerName1);
-
- docker.stopContainer(containerName2);
- docker.deleteContainer(containerName2);
- }
-
- @Test
- public void testContainerCycle() throws IOException, InterruptedException, ExecutionException {
- final ContainerName containerName = new ContainerName("docker-test-foo");
- final String containerHostname = "hostName1";
-
- docker.createContainerCommand(dockerImage, ContainerResources.UNLIMITED, containerName, containerHostname)
- .withManagedBy(MANAGER_NAME).create();
- Optional<Container> container = docker.getContainer(containerName);
- assertTrue(container.isPresent());
- assertEquals(container.get().state, Container.State.CREATED);
-
- docker.startContainer(containerName);
- container = docker.getContainer(containerName);
- assertTrue(container.isPresent());
- assertEquals(container.get().state, Container.State.RUNNING);
-
- docker.dockerClient.pauseContainerCmd(containerName.asString()).exec();
- container = docker.getContainer(containerName);
- assertTrue(container.isPresent());
- assertEquals(container.get().state, Container.State.PAUSED);
-
- docker.dockerClient.unpauseContainerCmd(containerName.asString()).exec();
- docker.stopContainer(containerName);
- container = docker.getContainer(containerName);
- assertTrue(container.isPresent());
- assertEquals(container.get().state, Container.State.EXITED);
-
- docker.deleteContainer(containerName);
- assertThat(docker.listAllContainersManagedBy(MANAGER_NAME).isEmpty(), is(true));
- }
-
- /**
- * Test the expected behavior for exec when it times out - it should throw an exception when it times out,
- * and before the process completes.
- *
- * The test timeout value is set quite high to avoid noise if screwdriver is slow but lower than the process time.
- */
- @Test(expected = DockerExecTimeoutException.class, timeout = 2000)
- public void testContainerExecHounorsTimeout() throws IOException, InterruptedException, ExecutionException {
- final ContainerName containerName = new ContainerName("docker-test-foo");
- final String containerHostname = "hostName1";
-
- docker.createContainerCommand(dockerImage, ContainerResources.UNLIMITED, containerName, containerHostname)
- .withManagedBy(MANAGER_NAME).create();
- docker.startContainer(containerName);
- docker.executeInContainerAsRoot(containerName, 1L, "sh", "-c", "sleep 5");
- }
-
- /**
- * Test the expected behavior for exec that completes before specified timeout - it should return when the process finishes and not
- * wait for the timeout. Some previous tests indicated that this was not behaving correctly.
- *
- * No timeout implies infinite timeout.
- *
- * The test timeout value is set quite high to avoid noise if screwdriver is slow
- */
- @Test(timeout = 4000)
- public void testContainerExecDoesNotBlockUntilTimeoutWhenCommandFinishesBeforeTimeout() throws IOException, InterruptedException, ExecutionException {
- final ContainerName containerName = new ContainerName("docker-test-foo");
- final String containerHostname = "hostName1";
-
- docker.createContainerCommand(dockerImage, ContainerResources.UNLIMITED, containerName, containerHostname)
- .withManagedBy(MANAGER_NAME).create();
- docker.startContainer(containerName);
- docker.executeInContainerAsRoot(containerName, 2L, "sh", "-c", "echo hei");
-
- // Also test that this is the behavoir when not specifying timeout
- docker.executeInContainerAsRoot(containerName,"sh", "-c", "echo hei");
- }
-
- @Test
- public void testDockerNetworking() throws InterruptedException, ExecutionException, IOException {
- String hostName1 = "docker10.test.yahoo.com";
- String hostName2 = "docker11.test.yahoo.com";
- ContainerName containerName1 = new ContainerName("docker-test-1");
- ContainerName containerName2 = new ContainerName("docker-test-2");
- InetAddress inetAddress1 = InetAddress.getByName("172.18.10.10");
- InetAddress inetAddress2 = InetAddress.getByName("172.18.10.11");
-
- docker.createContainerCommand(dockerImage, ContainerResources.UNLIMITED, containerName1, hostName1)
- .withManagedBy(MANAGER_NAME)
- .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withIpAddress(inetAddress1).create();
- docker.startContainer(containerName1);
-
- docker.createContainerCommand(dockerImage, ContainerResources.UNLIMITED, containerName2, hostName2)
- .withManagedBy(MANAGER_NAME)
- .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withIpAddress(inetAddress2).create();
- docker.startContainer(containerName2);
-
- testReachabilityFromHost("http://" + inetAddress1.getHostAddress() + "/ping");
- testReachabilityFromHost("http://" + inetAddress2.getHostAddress() + "/ping");
-
- String[] curlFromNodeToNode = new String[]{"curl", "-g", "http://" + inetAddress2.getHostAddress() + "/ping"};
- ProcessResult result = docker.executeInContainerAsRoot(containerName1, curlFromNodeToNode);
- assertThat("Could not reach " + containerName2.asString() + " from " + containerName1.asString(),
- result.getOutput(), is("pong\n"));
-
- docker.stopContainer(containerName1);
- docker.deleteContainer(containerName1);
-
- docker.stopContainer(containerName2);
- docker.deleteContainer(containerName2);
- }
-
- @Before
- public void setup() throws InterruptedException, ExecutionException, IOException {
- if (docker == null) {
- assumeTrue(DockerTestUtils.dockerDaemonIsPresent());
-
- docker = DockerTestUtils.getDocker();
- DockerTestUtils.buildSimpleHttpServerDockerImage(docker, dockerImage);
- }
-
- // Clean up any non deleted containers from previous tests
- docker.getAllContainersManagedBy(MANAGER_NAME).forEach(container -> {
- if (container.state.isRunning()) docker.stopContainer(container.name);
- docker.deleteContainer(container.name);
- });
- }
-
- private void testReachabilityFromHost(String target) throws IOException, InterruptedException {
- URL url = new URL(target);
- String containerServer = IOUtils.toString(url.openStream());
- assertThat(containerServer, is("pong\n"));
- }
-}
diff --git a/docker-api/src/test/resources/simple-ipv6-server/Dockerfile b/docker-api/src/test/resources/simple-ipv6-server/Dockerfile
deleted file mode 100644
index ee33894dbeb..00000000000
--- a/docker-api/src/test/resources/simple-ipv6-server/Dockerfile
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-FROM gliderlabs/alpine:3.4
-
-# Install python and curl
-RUN apk-install python curl
-
-# Copy source
-ADD src/ pysrc
-
-# Run http server on port 80
-EXPOSE 80
-CMD ["python", "/pysrc/server.py"]
diff --git a/docker-api/src/test/resources/simple-ipv6-server/README b/docker-api/src/test/resources/simple-ipv6-server/README
deleted file mode 100644
index 0cb96035c42..00000000000
--- a/docker-api/src/test/resources/simple-ipv6-server/README
+++ /dev/null
@@ -1,10 +0,0 @@
-Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-This is the source for a basic docker image that runs a python HTTP server listening at IPv6 port 80.
-The server serves two basic paths:
- /ip - returns IP address of the requester
- /ping - returns string "pong"
-
-
-To build the image run:
-$ sudo docker build -t "simple-ipv6-server:Dockerfile" <path to this directory>
diff --git a/docker-api/src/test/resources/simple-ipv6-server/src/fillmem.py b/docker-api/src/test/resources/simple-ipv6-server/src/fillmem.py
deleted file mode 100644
index b3990bea859..00000000000
--- a/docker-api/src/test/resources/simple-ipv6-server/src/fillmem.py
+++ /dev/null
@@ -1,11 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-import sys
-import time
-
-megabyte = [0] * (1024 * 1024 / 8)
-data = megabyte * int(sys.argv[1])
-
-while True:
- time.sleep(1)
- data.extend(megabyte)
diff --git a/docker-api/src/test/resources/simple-ipv6-server/src/server.py b/docker-api/src/test/resources/simple-ipv6-server/src/server.py
deleted file mode 100644
index 9b4d543d4ed..00000000000
--- a/docker-api/src/test/resources/simple-ipv6-server/src/server.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-import socket
-from BaseHTTPServer import HTTPServer
-from SimpleHTTPServer import SimpleHTTPRequestHandler
-
-
-class MyHandler(SimpleHTTPRequestHandler):
- def do_GET(self):
- if self.path == '/ip':
- self.send_response(200)
- self.send_header('Content-type', 'text/html')
- self.end_headers()
- self.wfile.write('Your IP address is %s\n' % self.client_address[0])
- return
-
- elif self.path == '/ping':
- self.send_response(200)
- self.send_header('Content-type', 'text/html')
- self.end_headers()
- self.wfile.write('pong\n')
- return
-
- else:
- self.send_response(404)
- self.send_header('Content-type', 'text/html')
- self.end_headers()
- self.wfile.write('Could not find ' + self.path + '! Try /ping or /ip.\n')
- return
-
-
-class DualHTTPServer(HTTPServer):
- def __init__(self, address, handler):
- self.address_family = socket.AF_INET6 if (':' in address[0]) else socket.AF_INET
- HTTPServer.__init__(self, address, handler)
-
-
-def main(ipv6):
- server = DualHTTPServer(('::' if ipv6 else '', 80), MyHandler)
- server.serve_forever()
-
-
-if __name__ == '__main__':
- main(False)