move identityprovider package to vespa-athenz

14 files changed, 1074 insertions, 1 deletions

diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml
index 8d83a14fcbd..31e56f76dd2 100644
--- a/vespa-athenz/pom.xml
+++ b/vespa-athenz/pom.xml
@@ -41,7 +41,12 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
-
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>testutil</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
 
         <!-- compile -->
         <dependency>
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzCredentials.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzCredentials.java
new file mode 100644
index 00000000000..c5dce1c5b1d
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzCredentials.java
@@ -0,0 +1,51 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
+import java.time.Instant;
+
+/**
+ * @author bjorncs
+ */
+class AthenzCredentials {
+
+    private final String nToken;
+    private final X509Certificate certificate;
+    private final KeyPair keyPair;
+    private final SignedIdentityDocument identityDocument;
+    private final Instant createdAt;
+
+    AthenzCredentials(String nToken,
+                      X509Certificate certificate,
+                      KeyPair keyPair,
+                      SignedIdentityDocument identityDocument,
+                      Instant createdAt) {
+        this.nToken = nToken;
+        this.certificate = certificate;
+        this.keyPair = keyPair;
+        this.identityDocument = identityDocument;
+        this.createdAt = createdAt;
+    }
+
+    String getNToken() {
+        return nToken;
+    }
+
+    X509Certificate getCertificate() {
+        return certificate;
+    }
+
+    KeyPair getKeyPair() {
+        return keyPair;
+    }
+
+    SignedIdentityDocument getIdentityDocument() {
+        return identityDocument;
+    }
+
+    Instant getCreatedAt() {
+        return createdAt;
+    }
+
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzCredentialsService.java
new file mode 100644
index 00000000000..dd816929bfb
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzCredentialsService.java
@@ -0,0 +1,93 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.container.core.identity.IdentityConfig;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
+import java.time.Clock;
+
+/**
+ * @author bjorncs
+ */
+class AthenzCredentialsService {
+
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    private final IdentityConfig identityConfig;
+    private final IdentityDocumentService identityDocumentService;
+    private final AthenzService athenzService;
+    private final Clock clock;
+
+    AthenzCredentialsService(IdentityConfig identityConfig,
+                             IdentityDocumentService identityDocumentService,
+                             AthenzService athenzService,
+                             Clock clock) {
+        this.identityConfig = identityConfig;
+        this.identityDocumentService = identityDocumentService;
+        this.athenzService = athenzService;
+        this.clock = clock;
+    }
+
+    AthenzCredentials registerInstance() {
+        KeyPair keyPair = CryptoUtils.createKeyPair();
+        String rawDocument = identityDocumentService.getSignedIdentityDocument();
+        SignedIdentityDocument document = parseSignedIdentityDocument(rawDocument);
+        PKCS10CertificationRequest csr = CryptoUtils.createCSR(identityConfig.domain(),
+                                                               identityConfig.service(),
+                                                               document.dnsSuffix,
+                                                               document.providerUniqueId,
+                                                               keyPair);
+        InstanceRegisterInformation instanceRegisterInformation =
+                new InstanceRegisterInformation(document.providerService,
+                                                identityConfig.domain(),
+                                                identityConfig.service(),
+                                                rawDocument,
+                                                CryptoUtils.toPem(csr));
+        InstanceIdentity instanceIdentity = athenzService.sendInstanceRegisterRequest(instanceRegisterInformation,
+                                                                                      document.ztsEndpoint);
+        return toAthenzCredentials(instanceIdentity, keyPair, document);
+    }
+
+    AthenzCredentials updateCredentials(AthenzCredentials currentCredentials) {
+        SignedIdentityDocument document = currentCredentials.getIdentityDocument();
+        KeyPair newKeyPair = CryptoUtils.createKeyPair();
+        PKCS10CertificationRequest csr = CryptoUtils.createCSR(identityConfig.domain(),
+                                                               identityConfig.service(),
+                                                               document.dnsSuffix,
+                                                               document.providerUniqueId,
+                                                               newKeyPair);
+        InstanceRefreshInformation refreshInfo = new InstanceRefreshInformation(CryptoUtils.toPem(csr));
+        InstanceIdentity instanceIdentity =
+                athenzService.sendInstanceRefreshRequest(document.providerService,
+                                                         identityConfig.domain(),
+                                                         identityConfig.service(),
+                                                         document.providerUniqueId,
+                                                         refreshInfo,
+                                                         document.ztsEndpoint,
+                                                         currentCredentials.getCertificate(),
+                                                         currentCredentials.getKeyPair().getPrivate());
+        return toAthenzCredentials(instanceIdentity, newKeyPair, document);
+    }
+
+    private AthenzCredentials toAthenzCredentials(InstanceIdentity instanceIdentity,
+                                                         KeyPair keyPair,
+                                                         SignedIdentityDocument identityDocument) {
+        X509Certificate certificate = instanceIdentity.getX509Certificate();
+        String serviceToken = instanceIdentity.getServiceToken();
+        return new AthenzCredentials(serviceToken, certificate, keyPair, identityDocument, clock.instant());
+    }
+
+    private static SignedIdentityDocument parseSignedIdentityDocument(String rawDocument) {
+        try {
+            return mapper.readValue(rawDocument, SignedIdentityDocument.class);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzIdentityProviderImpl.java
new file mode 100644
index 00000000000..95113e1b0b1
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzIdentityProviderImpl.java
@@ -0,0 +1,224 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.container.core.identity.IdentityConfig;
+import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider;
+import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.athenz.api.AthenzIdentityCertificate;
+import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder;
+import com.yahoo.vespa.defaults.Defaults;
+
+import javax.net.ssl.SSLContext;
+import java.io.File;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Logger;
+
+/**
+ * @author mortent
+ * @author bjorncs
+ */
+public final class AthenzIdentityProviderImpl extends AbstractComponent implements AthenzIdentityProvider {
+
+    private static final Logger log = Logger.getLogger(AthenzIdentityProviderImpl.class.getName());
+
+    // TODO Make some of these values configurable through config. Match requested expiration of register/update requests.
+    // TODO These should match the requested expiration
+    static final Duration EXPIRES_AFTER = Duration.ofDays(1);
+    static final Duration EXPIRATION_MARGIN = Duration.ofMinutes(30);
+    static final Duration INITIAL_WAIT_NTOKEN = Duration.ofMinutes(5);
+    static final Duration UPDATE_PERIOD = EXPIRES_AFTER.dividedBy(3);
+    static final Duration REDUCED_UPDATE_PERIOD = Duration.ofMinutes(30);
+    static final Duration INITIAL_BACKOFF_DELAY = Duration.ofMinutes(4);
+    static final Duration MAX_REGISTER_BACKOFF_DELAY = Duration.ofHours(1);
+    static final int BACKOFF_DELAY_MULTIPLIER = 2;
+    static final Duration AWAIT_TERMINTATION_TIMEOUT = Duration.ofSeconds(90);
+
+    private static final Duration CERTIFICATE_EXPIRY_METRIC_UPDATE_PERIOD = Duration.ofMinutes(5);
+    private static final String CERTIFICATE_EXPIRY_METRIC_NAME = "athenz-tenant-cert.expiry.seconds";
+
+    static final String REGISTER_INSTANCE_TAG = "register-instance";
+    static final String UPDATE_CREDENTIALS_TAG = "update-credentials";
+    static final String TIMEOUT_INITIAL_WAIT_TAG = "timeout-initial-wait";
+    static final String METRICS_UPDATER_TAG = "metrics-updater";
+
+
+    private volatile AthenzCredentials credentials;
+    private final AtomicReference<Throwable> lastThrowable = new AtomicReference<>();
+    private final AthenzCredentialsService athenzCredentialsService;
+    private final Scheduler scheduler;
+    private final Clock clock;
+    private final String domain;
+    private final String service;
+
+    private final CertificateExpiryMetricUpdater metricUpdater;
+
+    @Inject
+    public AthenzIdentityProviderImpl(IdentityConfig config, Metric metric) {
+        this(config,
+             metric,
+             new AthenzCredentialsService(config,
+                                          new IdentityDocumentService(config.loadBalancerAddress()),
+                                          new AthenzService(),
+                                          Clock.systemUTC()),
+             new ThreadPoolScheduler(),
+             Clock.systemUTC());
+    }
+
+    // Test only
+    AthenzIdentityProviderImpl(IdentityConfig config,
+                               Metric metric,
+                               AthenzCredentialsService athenzCredentialsService,
+                               Scheduler scheduler,
+                               Clock clock) {
+        this.athenzCredentialsService = athenzCredentialsService;
+        this.scheduler = scheduler;
+        this.clock = clock;
+        this.domain = config.domain();
+        this.service = config.service();
+        metricUpdater = new CertificateExpiryMetricUpdater(metric);
+        registerInstance();
+    }
+
+    private void registerInstance() {
+        try {
+            credentials = athenzCredentialsService.registerInstance();
+            scheduler.schedule(new UpdateCredentialsTask(), UPDATE_PERIOD);
+            scheduler.submit(metricUpdater);
+        } catch (Throwable t) {
+            throw new AthenzIdentityProviderException("Could not retrieve Athenz credentials", t);
+        }
+    }
+
+    @Override
+    public String getDomain() {
+        return domain;
+    }
+
+    @Override
+    public String getService() {
+        return service;
+    }
+
+    @Override
+    public SSLContext getIdentitySslContext() {
+        return new AthenzSslContextBuilder()
+                .withIdentityCertificate(new AthenzIdentityCertificate(
+                        credentials.getCertificate(),
+                        credentials.getKeyPair().getPrivate()))
+                .withTrustStore(new File(Defaults.getDefaults().underVespaHome("share/ssl/certs/yahoo_certificate_bundle.jks")), "JKS")
+                .build();
+    }
+
+    @Override
+    public void deconstruct() {
+        scheduler.shutdown(AWAIT_TERMINTATION_TIMEOUT);
+    }
+
+    private boolean isExpired(AthenzCredentials credentials) {
+        return clock.instant().isAfter(getExpirationTime(credentials));
+    }
+
+    private static Instant getExpirationTime(AthenzCredentials credentials) {
+        return credentials.getCreatedAt().plus(EXPIRES_AFTER).minus(EXPIRATION_MARGIN);
+    }
+
+    private class UpdateCredentialsTask implements RunnableWithTag {
+        @Override
+        public void run() {
+            try {
+                AthenzCredentials newCredentials = isExpired(credentials)
+                        ? athenzCredentialsService.registerInstance()
+                        : athenzCredentialsService.updateCredentials(credentials);
+                credentials = newCredentials;
+                scheduler.schedule(new UpdateCredentialsTask(), UPDATE_PERIOD);
+            } catch (Throwable t) {
+                log.log(LogLevel.WARNING, "Failed to update credentials: " + t.getMessage(), t);
+                lastThrowable.set(t);
+                Duration timeToExpiration = Duration.between(clock.instant(), getExpirationTime(credentials));
+                // NOTE: Update period might be after timeToExpiration, still we do not want to DDoS Athenz.
+                Duration updatePeriod =
+                        timeToExpiration.compareTo(UPDATE_PERIOD) > 0 ? UPDATE_PERIOD : REDUCED_UPDATE_PERIOD;
+                scheduler.schedule(new UpdateCredentialsTask(), updatePeriod);
+            }
+        }
+
+        @Override
+        public String tag() {
+            return UPDATE_CREDENTIALS_TAG;
+        }
+    }
+
+    private class CertificateExpiryMetricUpdater implements RunnableWithTag {
+        private final Metric metric;
+
+        private CertificateExpiryMetricUpdater(Metric metric) {
+            this.metric = metric;
+        }
+
+        @Override
+        public void run() {
+            Instant expirationTime = getExpirationTime(credentials);
+            Duration remainingLifetime = Duration.between(clock.instant(), expirationTime);
+            metric.set(CERTIFICATE_EXPIRY_METRIC_NAME, remainingLifetime.getSeconds(), null);
+            scheduler.schedule(this, CERTIFICATE_EXPIRY_METRIC_UPDATE_PERIOD);
+        }
+
+        @Override
+        public String tag() {
+            return METRICS_UPDATER_TAG;
+        }
+    }
+
+    private static class ThreadPoolScheduler implements Scheduler {
+
+        private static final Logger log = Logger.getLogger(ThreadPoolScheduler.class.getName());
+
+        private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(0);
+
+        @Override
+        public void schedule(RunnableWithTag runnable, Duration delay) {
+            log.log(LogLevel.FINE, String.format("Scheduling task '%s' in '%s'", runnable.tag(), delay));
+            executor.schedule(runnable, delay.getSeconds(), TimeUnit.SECONDS);
+        }
+
+        @Override
+        public void submit(RunnableWithTag runnable) {
+            log.log(LogLevel.FINE, String.format("Scheduling task '%s' now", runnable.tag()));
+            executor.submit(runnable);
+        }
+
+        @Override
+        public void shutdown(Duration timeout) {
+            try {
+                executor.shutdownNow();
+                executor.awaitTermination(AWAIT_TERMINTATION_TIMEOUT.getSeconds(), TimeUnit.SECONDS);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+    }
+
+    public interface Scheduler {
+        void schedule(RunnableWithTag runnable, Duration delay);
+        default void submit(RunnableWithTag runnable) { schedule(runnable, Duration.ZERO); }
+        default void shutdown(Duration timeout) {}
+    }
+
+    public interface RunnableWithTag extends Runnable {
+
+        String tag();
+    }
+
+}
+
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzService.java
new file mode 100644
index 00000000000..18576ab9bab
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzService.java
@@ -0,0 +1,124 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.conn.ssl.SSLContextBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+import org.eclipse.jetty.http.HttpStatus;
+
+import javax.net.ssl.SSLContext;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+/**
+ * @author mortent
+ * @author bjorncs
+ */
+public class AthenzService {
+
+    private static final String INSTANCE_API_PATH = "/zts/v1/instance";
+
+    private final ObjectMapper objectMapper = new ObjectMapper();
+    private final HttpRequestRetryHandler retryHandler = new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true);
+
+    /**
+     * Send instance register request to ZTS, get InstanceIdentity
+     */
+     public InstanceIdentity sendInstanceRegisterRequest(InstanceRegisterInformation instanceRegisterInformation,
+                                                         URI uri) {
+        try(CloseableHttpClient client = HttpClientBuilder.create().setRetryHandler(retryHandler).build()) {
+            HttpUriRequest postRequest = RequestBuilder.post()
+                    .setUri(uri.resolve(INSTANCE_API_PATH))
+                    .setEntity(toJsonStringEntity(instanceRegisterInformation))
+                    .build();
+            return getInstanceIdentity(client, postRequest);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public InstanceIdentity sendInstanceRefreshRequest(String providerService,
+                                                       String instanceDomain,
+                                                       String instanceServiceName,
+                                                       String instanceId,
+                                                       InstanceRefreshInformation instanceRefreshInformation,
+                                                       URI ztsEndpoint,
+                                                       X509Certificate certicate,
+                                                       PrivateKey privateKey) {
+        try (CloseableHttpClient client = createHttpClientWithTlsAuth(certicate, privateKey, retryHandler)) {
+            URI uri = ztsEndpoint
+                    .resolve(INSTANCE_API_PATH + '/')
+                    .resolve(providerService + '/')
+                    .resolve(instanceDomain + '/')
+                    .resolve(instanceServiceName + '/')
+                    .resolve(instanceId);
+            HttpUriRequest postRequest = RequestBuilder.post()
+                    .setUri(uri)
+                    .setEntity(toJsonStringEntity(instanceRefreshInformation))
+                    .build();
+            return getInstanceIdentity(client, postRequest);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private InstanceIdentity getInstanceIdentity(CloseableHttpClient client, HttpUriRequest postRequest)
+            throws IOException {
+        try (CloseableHttpResponse response = client.execute(postRequest)) {
+            if(HttpStatus.isSuccess(response.getStatusLine().getStatusCode())) {
+                return objectMapper.readValue(response.getEntity().getContent(), InstanceIdentity.class);
+            } else {
+                String message = EntityUtils.toString(response.getEntity());
+                throw new RuntimeException(String.format("Unable to get identity. http code/message: %d/%s",
+                                                         response.getStatusLine().getStatusCode(), message));
+            }
+        }
+    }
+
+    private StringEntity toJsonStringEntity(Object value) throws JsonProcessingException {
+        return new StringEntity(objectMapper.writeValueAsString(value), ContentType.APPLICATION_JSON);
+    }
+
+    private static CloseableHttpClient createHttpClientWithTlsAuth(X509Certificate certificate,
+                                                                   PrivateKey privateKey,
+                                                                   HttpRequestRetryHandler retryHandler) {
+        try {
+            String dummyPassword = "athenz";
+            KeyStore keyStore = KeyStore.getInstance("JKS");
+            keyStore.load(null);
+            keyStore.setKeyEntry("athenz", privateKey, dummyPassword.toCharArray(), new Certificate[]{certificate});
+            SSLContext sslContext = new SSLContextBuilder()
+                    .loadKeyMaterial(keyStore, dummyPassword.toCharArray())
+                    .build();
+            return HttpClientBuilder.create()
+                    .setRetryHandler(retryHandler)
+                    .setSslcontext(sslContext)
+                    .build();
+        } catch (KeyStoreException | UnrecoverableKeyException | NoSuchAlgorithmException |
+                KeyManagementException | CertificateException e) {
+            throw new RuntimeException(e);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/CryptoUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/CryptoUtils.java
new file mode 100644
index 00000000000..6e74d3bc8b1
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/CryptoUtils.java
@@ -0,0 +1,113 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.ExtensionsGenerator;
+import org.bouncycastle.asn1.x509.GeneralName;
+import org.bouncycastle.asn1.x509.GeneralNames;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.openssl.PEMParser;
+import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
+import org.bouncycastle.util.io.pem.PemObject;
+
+import javax.security.auth.x500.X500Principal;
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.io.UncheckedIOException;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+/**
+ * @author bjorncs
+ */
+class CryptoUtils {
+
+    private static final BouncyCastleProvider bouncyCastleProvider = new BouncyCastleProvider();
+
+    private CryptoUtils() {}
+
+    static KeyPair createKeyPair() {
+        try {
+            KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA");
+            return kpg.generateKeyPair();
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static PKCS10CertificationRequest createCSR(String identityDomain,
+                                                String identityService,
+                                                String dnsSuffix,
+                                                String providerUniqueId,
+                                                KeyPair keyPair) {
+        try {
+            // Add SAN dnsname <service>.<domain-with-dashes>.<provider-dnsname-suffix>
+            // and SAN dnsname <provider-unique-instance-id>.instanceid.athenz.<provider-dnsname-suffix>
+            GeneralNames subjectAltNames = new GeneralNames(new GeneralName[]{
+                    new GeneralName(GeneralName.dNSName, String.format("%s.%s.%s",
+                                                                       identityService,
+                                                                       identityDomain.replace(".", "-"),
+                                                                       dnsSuffix)),
+                    new GeneralName(GeneralName.dNSName, String.format("%s.instanceid.athenz.%s",
+                                                                       providerUniqueId,
+                                                                       dnsSuffix))
+            });
+
+            ExtensionsGenerator extGen = new ExtensionsGenerator();
+            extGen.addExtension(Extension.subjectAlternativeName, false, subjectAltNames);
+
+            X500Principal subject = new X500Principal(
+                    String.format("CN=%s.%s", identityDomain, identityService));
+
+            PKCS10CertificationRequestBuilder requestBuilder =
+                    new JcaPKCS10CertificationRequestBuilder(subject, keyPair.getPublic());
+            requestBuilder.addAttribute(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extGen.generate());
+            return requestBuilder.build(new JcaContentSignerBuilder("SHA256withRSA").build(keyPair.getPrivate()));
+        } catch (OperatorCreationException e) {
+            throw new RuntimeException(e);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    static String toPem(PKCS10CertificationRequest csr) {
+        try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) {
+            pemWriter.writeObject(new PemObject("CERTIFICATE REQUEST", csr.getEncoded()));
+            pemWriter.flush();
+            return stringWriter.toString();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    static X509Certificate parseCertificate(String pemEncodedCertificate) {
+        try (PEMParser parser = new PEMParser(new StringReader(pemEncodedCertificate))) {
+            Object pemObject = parser.readObject();
+            if (pemObject instanceof X509Certificate) {
+                return (X509Certificate) pemObject;
+            }
+            if (pemObject instanceof X509CertificateHolder) {
+                return new JcaX509CertificateConverter()
+                        .setProvider(bouncyCastleProvider)
+                        .getCertificate((X509CertificateHolder) pemObject);
+            }
+            throw new IllegalArgumentException("Invalid type of PEM object: " + pemObject);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        } catch (CertificateException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/IdentityDocumentService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/IdentityDocumentService.java
new file mode 100644
index 00000000000..4e88234d5de
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/IdentityDocumentService.java
@@ -0,0 +1,85 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import com.yahoo.vespa.defaults.Defaults;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLContextBuilder;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+import org.eclipse.jetty.http.HttpStatus;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * @author mortent
+ * @author bjorncs
+ */
+public class IdentityDocumentService {
+
+    private final URI identityDocumentApiUri;
+
+    public IdentityDocumentService(String loadBalancerName) {
+        this.identityDocumentApiUri = createIdentityDocumentApiUri(loadBalancerName);
+    }
+
+    /**
+     * Get signed identity document from config server
+     */
+    public String getSignedIdentityDocument() {
+        try (CloseableHttpClient httpClient = createHttpClient()) {
+            CloseableHttpResponse idDocResponse = httpClient.execute(new HttpGet(identityDocumentApiUri));
+            String responseContent = EntityUtils.toString(idDocResponse.getEntity());
+            if (HttpStatus.isSuccess(idDocResponse.getStatusLine().getStatusCode())) {
+                return responseContent;
+            } else {
+                // TODO make sure we have retried a few times (AND logged) before giving up
+                throw new RuntimeException(
+                        "Failed to initialize Athenz instance provider: " +
+                                idDocResponse.getStatusLine() + ": " + responseContent);
+            }
+        } catch (IOException e) {
+            throw new RuntimeException("Failed getting signed identity document", e);
+        }
+    }
+
+    // TODO Use client side auth to establish trusted secure channel
+    // TODO Validate TLS certifcate of config server
+    private static CloseableHttpClient createHttpClient() {
+        try {
+            SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
+            sslContextBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
+            SSLConnectionSocketFactory sslSocketFactory =
+                    new SSLConnectionSocketFactory(sslContextBuilder.build(),
+                                                   SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+            return HttpClientBuilder.create().setSSLSocketFactory(sslSocketFactory).setUserAgent("identity-document-client").build();
+        } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static URI createIdentityDocumentApiUri(String loadBalancerName) {
+        try {
+            // TODO Figure out a proper way of determining the hostname matching what's registred in node-repository
+            return new URIBuilder()
+                    .setScheme("https")
+                    .setHost(loadBalancerName)
+                    .setPort(4443)
+                    .setPath("/athenz/v1/provider/identity-document")
+                    .addParameter("hostname", Defaults.getDefaults().vespaHostname())
+                    .build();
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/InstanceIdentity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/InstanceIdentity.java
new file mode 100644
index 00000000000..b90ce56ca7e
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/InstanceIdentity.java
@@ -0,0 +1,48 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.io.IOException;
+import java.security.cert.X509Certificate;
+
+/**
+ * Used for deserializing response from ZTS
+ *
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class InstanceIdentity {
+    @JsonProperty("x509Certificate") private final X509Certificate x509Certificate;
+    @JsonProperty("serviceToken") private final String serviceToken;
+
+    public InstanceIdentity(@JsonProperty("x509Certificate") @JsonDeserialize(using = X509CertificateDeserializer.class)
+                                    X509Certificate x509Certificate,
+                            @JsonProperty("serviceToken") String serviceToken) {
+        this.x509Certificate = x509Certificate;
+        this.serviceToken = serviceToken;
+    }
+
+    public X509Certificate getX509Certificate() {
+        return x509Certificate;
+    }
+
+    public String getServiceToken() {
+        return serviceToken;
+    }
+
+    public static class X509CertificateDeserializer extends JsonDeserializer<X509Certificate> {
+        @Override
+        public X509Certificate deserialize(JsonParser parser, DeserializationContext context) throws IOException {
+            return CryptoUtils.parseCertificate(parser.getValueAsString());
+        }
+    }
+
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/InstanceRefreshInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/InstanceRefreshInformation.java
new file mode 100644
index 00000000000..c627363c0f5
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/InstanceRefreshInformation.java
@@ -0,0 +1,23 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author bjorncs
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class InstanceRefreshInformation {
+
+    @JsonProperty("csr")
+    private final String csr;
+    @JsonProperty("token")
+    private final boolean requestServiceToken = true;
+
+    public InstanceRefreshInformation(String csr) {
+        this.csr = csr;
+    }
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/InstanceRegisterInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/InstanceRegisterInformation.java
new file mode 100644
index 00000000000..69ddb72b8b8
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/InstanceRegisterInformation.java
@@ -0,0 +1,38 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Used for serializing request to ZTS
+ *
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class InstanceRegisterInformation {
+    @JsonProperty("provider")
+    private final String provider;
+    @JsonProperty("domain")
+    private final String domain;
+    @JsonProperty("service")
+    private final String service;
+    @JsonProperty("attestationData")
+    private final String attestationData;
+    @JsonProperty("ssh")
+    private final String ssh = null; // Not needed
+    @JsonProperty("csr")
+    private final String csr;
+    @JsonProperty("token")
+    private final boolean token = true;
+
+    public InstanceRegisterInformation(String provider, String domain, String service, String attestationData, String csr) {
+        this.provider = provider;
+        this.domain = domain;
+        this.service = service;
+        this.attestationData = attestationData;
+        this.csr = csr;
+    }
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/SignedIdentityDocument.java
new file mode 100644
index 00000000000..c3b073765ac
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/SignedIdentityDocument.java
@@ -0,0 +1,33 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.net.URI;
+
+/**
+ * @author bjorncs
+ */
+// TODO Most of these value should ideally be config provided by config-model
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+class SignedIdentityDocument {
+    public final String providerUniqueId;
+    public final String dnsSuffix;
+    public final String providerService;
+    public final URI ztsEndpoint;
+
+    public SignedIdentityDocument(@JsonProperty("provider-unique-id") String providerUniqueId,
+                                  @JsonProperty("dns-suffix") String dnsSuffix,
+                                  @JsonProperty("provider-service") String providerService,
+                                  @JsonProperty("zts-endpoint") URI ztsEndpoint) {
+        this.providerUniqueId = providerUniqueId;
+        this.dnsSuffix = dnsSuffix;
+        this.providerService = providerService;
+        this.ztsEndpoint = ztsEndpoint;
+    }
+
+}
+
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/package-info.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/package-info.java
new file mode 100644
index 00000000000..f23ea9406b3
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author mortent
+ */
+@ExportPackage
+package com.yahoo.vespa.athenz.identityprovider;
+
+import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file
diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/AthenzIdentityProviderImplTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/AthenzIdentityProviderImplTest.java
new file mode 100644
index 00000000000..d9dbd73a94e
--- /dev/null
+++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/AthenzIdentityProviderImplTest.java
@@ -0,0 +1,200 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import com.yahoo.container.core.identity.IdentityConfig;
+import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider;
+import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.athenz.identityprovider.AthenzIdentityProviderImpl.RunnableWithTag;
+import com.yahoo.vespa.athenz.identityprovider.AthenzIdentityProviderImpl.Scheduler;
+import org.junit.Test;
+
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.PriorityQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+import static com.yahoo.vespa.athenz.identityprovider.AthenzIdentityProviderImpl.METRICS_UPDATER_TAG;
+import static com.yahoo.vespa.athenz.identityprovider.AthenzIdentityProviderImpl.REDUCED_UPDATE_PERIOD;
+import static com.yahoo.vespa.athenz.identityprovider.AthenzIdentityProviderImpl.UPDATE_CREDENTIALS_TAG;
+import static com.yahoo.vespa.athenz.identityprovider.AthenzIdentityProviderImpl.UPDATE_PERIOD;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author mortent
+ * @author bjorncs
+ */
+public class AthenzIdentityProviderImplTest {
+
+    private static final Metric DUMMY_METRIC = new Metric() {
+        @Override
+        public void set(String s, Number number, Context context) {
+        }
+
+        @Override
+        public void add(String s, Number number, Context context) {
+        }
+
+        @Override
+        public Context createContext(Map<String, ?> stringMap) {
+            return null;
+        }
+    };
+
+    private static final IdentityConfig IDENTITY_CONFIG =
+            new IdentityConfig(new IdentityConfig.Builder()
+                                       .service("tenantService").domain("tenantDomain").loadBalancerAddress("cfg"));
+
+    @Test (expected = AthenzIdentityProviderException.class)
+    public void component_creation_fails_when_credentials_not_found() {
+        AthenzCredentialsService credentialService = mock(AthenzCredentialsService.class);
+        when(credentialService.registerInstance())
+                .thenThrow(new RuntimeException("athenz unavailable"));
+
+        ManualClock clock = new ManualClock(Instant.EPOCH);
+        MockScheduler scheduler = new MockScheduler(clock);
+        AthenzIdentityProvider identityProvider =
+                new AthenzIdentityProviderImpl(IDENTITY_CONFIG, DUMMY_METRIC, credentialService, scheduler, clock);
+    }
+
+    @Test
+    public void failed_credentials_updates_will_schedule_retries() {
+        IdentityDocumentService identityDocumentService = mock(IdentityDocumentService.class);
+        AthenzService athenzService = mock(AthenzService.class);
+        ManualClock clock = new ManualClock(Instant.EPOCH);
+        MockScheduler scheduler = new MockScheduler(clock);
+        X509Certificate x509Certificate = mock(X509Certificate.class);
+
+        when(identityDocumentService.getSignedIdentityDocument()).thenReturn(getIdentityDocument());
+        when(athenzService.sendInstanceRegisterRequest(any(), any())).thenReturn(
+                new InstanceIdentity(null, "TOKEN"));
+        when(athenzService.sendInstanceRefreshRequest(anyString(), anyString(), anyString(),
+                                                      anyString(), any(), any(), any(), any()))
+                .thenThrow(new RuntimeException("#1"))
+                .thenThrow(new RuntimeException("#2"))
+                .thenThrow(new RuntimeException("#3"))
+                .thenReturn(new InstanceIdentity(null, "TOKEN"));
+        AthenzCredentialsService credentialService =
+                new AthenzCredentialsService(IDENTITY_CONFIG, identityDocumentService, athenzService, clock);
+
+        AthenzIdentityProvider identityProvider =
+                new AthenzIdentityProviderImpl(IDENTITY_CONFIG, DUMMY_METRIC, credentialService, scheduler, clock);
+
+        List<MockScheduler.CompletedTask> expectedTasks =
+                Arrays.asList(
+                        new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, UPDATE_PERIOD),
+                        new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, UPDATE_PERIOD),
+                        new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, REDUCED_UPDATE_PERIOD),
+                        new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, REDUCED_UPDATE_PERIOD),
+                        new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, UPDATE_PERIOD));
+        AtomicInteger counter = new AtomicInteger(0);
+        List<MockScheduler.CompletedTask> completedTasks =
+                scheduler.runAllTasks(task -> !task.tag().equals(METRICS_UPDATER_TAG) &&
+                                              counter.getAndIncrement() < expectedTasks.size());
+        assertEquals(expectedTasks, completedTasks);
+    }
+
+    private static String getIdentityDocument() {
+        return "{\n" +
+               "  \"identity-document\": \"eyJwcm92aWRlci11bmlxdWUtaWQiOnsidGVuYW50IjoidGVuYW50IiwiYXBwbGljYXRpb24iOiJhcHBsaWNhdGlvbiIsImVudmlyb25tZW50IjoiZGV2IiwicmVnaW9uIjoidXMtbm9ydGgtMSIsImluc3RhbmNlIjoiZGVmYXVsdCIsImNsdXN0ZXItaWQiOiJkZWZhdWx0IiwiY2x1c3Rlci1pbmRleCI6MH0sImNvbmZpZ3NlcnZlci1ob3N0bmFtZSI6ImxvY2FsaG9zdCIsImluc3RhbmNlLWhvc3RuYW1lIjoieC55LmNvbSIsImNyZWF0ZWQtYXQiOjE1MDg3NDgyODUuNzQyMDAwMDAwfQ==\",\n" +
+               "  \"signature\": \"kkEJB/98cy1FeXxzSjtvGH2a6BFgZu/9/kzCcAqRMZjENxnw5jyO1/bjZVzw2Sz4YHPsWSx2uxb32hiQ0U8rMP0zfA9nERIalSP0jB/hMU8laezGhdpk6VKZPJRC6YKAB9Bsv2qUIfMsSxkMqf66GUvjZAGaYsnNa2yHc1jIYHOGMeJO+HNPYJjGv26xPfAOPIKQzs3RmKrc3FoweTCsIwm5oblqekdJvVWYe0obwlOSB5uwc1zpq3Ie1QBFtJRuCGMVHg1pDPxXKBHLClGIrEvzLmICy6IRdHszSO5qiwujUD7sbrbM0sB/u0cYucxbcsGRUmBvme3UAw2mW9POVQ==\",\n" +
+               "  \"signing-key-version\": 0,\n" +
+               "  \"provider-unique-id\": \"tenant.application.dev.us-north-1.default.default.0\",\n" +
+               "  \"dns-suffix\": \"dnsSuffix\",\n" +
+               "  \"provider-service\": \"service\",\n" +
+               "  \"zts-endpoint\": \"localhost/zts\", \n" +
+               "  \"document-version\": 1\n" +
+               "}";
+
+    }
+
+    private static class MockScheduler implements Scheduler {
+
+        private final PriorityQueue<DelayedTask> tasks = new PriorityQueue<>();
+        private final ManualClock clock;
+
+        MockScheduler(ManualClock clock) {
+            this.clock = clock;
+        }
+
+        @Override
+        public void schedule(RunnableWithTag task, Duration delay) {
+            tasks.offer(new DelayedTask(task, delay, clock.instant().plus(delay)));
+        }
+
+        List<CompletedTask> runAllTasks(Predicate<RunnableWithTag> filter) {
+            List<CompletedTask> completedTasks = new ArrayList<>();
+            while (!tasks.isEmpty()) {
+                DelayedTask task = tasks.poll();
+                RunnableWithTag runnable = task.runnableWithTag;
+                if (filter.test(runnable)) {
+                    clock.setInstant(task.startTime);
+                    runnable.run();
+                    completedTasks.add(new CompletedTask(runnable.tag(), task.delay));
+                }
+            }
+            return completedTasks;
+        }
+
+        private static class DelayedTask implements Comparable<DelayedTask> {
+            final RunnableWithTag runnableWithTag;
+            final Duration delay;
+            final Instant startTime;
+
+            DelayedTask(RunnableWithTag runnableWithTag, Duration delay, Instant startTime) {
+                this.runnableWithTag = runnableWithTag;
+                this.delay = delay;
+                this.startTime = startTime;
+            }
+
+            @Override
+            public int compareTo(DelayedTask other) {
+                return this.startTime.compareTo(other.startTime);
+            }
+        }
+
+        private static class CompletedTask {
+            final String tag;
+            final Duration delay;
+
+            CompletedTask(String tag, Duration delay) {
+                this.tag = tag;
+                this.delay = delay;
+            }
+
+            @Override
+            public boolean equals(Object o) {
+                if (this == o) return true;
+                if (o == null || getClass() != o.getClass()) return false;
+                CompletedTask that = (CompletedTask) o;
+                return Objects.equals(tag, that.tag) &&
+                       Objects.equals(delay, that.delay);
+            }
+
+            @Override
+            public int hashCode() {
+                return Objects.hash(tag, delay);
+            }
+
+            @Override
+            public String toString() {
+                return "CompletedTask{" +
+                       "tag='" + tag + '\'' +
+                       ", delay=" + delay +
+                       '}';
+            }
+        }
+    }
+}
diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/CryptoUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/CryptoUtilsTest.java
new file mode 100644
index 00000000000..353c5d3c504
--- /dev/null
+++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/CryptoUtilsTest.java
@@ -0,0 +1,28 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider;
+
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.security.KeyPair;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author bjorncs
+ */
+public class CryptoUtilsTest {
+
+    @Test
+    public void certificate_signing_request_is_correct_and_can_be_serialized_to_pem() throws IOException {
+        KeyPair keyPair = CryptoUtils.createKeyPair();
+        PKCS10CertificationRequest csr = CryptoUtils.createCSR(
+                "identity-domain", "identity-service", "vespa.cloud.com", "unique.instance.id", keyPair);
+        String pem = CryptoUtils.toPem(csr);
+        assertThat(pem, containsString("BEGIN CERTIFICATE REQUEST"));
+        assertThat(pem, containsString("END CERTIFICATE REQUEST"));
+    }
+
+}