Use mutual TLS auth when communicating with ZTS

- Remove instance register/refresh from ad-hoc ZtsClient implementation
- Deprecate ad-hoc ZtsClient

7 files changed, 57 insertions, 238 deletions

diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java
index 270954c73b2..43378b6507a 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java
@@ -26,7 +26,7 @@ public interface ZtsClient extends AutoCloseable {
      */
     InstanceIdentity registerInstance(AthenzService providerIdentity,
                                       AthenzService instanceIdentity,
-                                      String instanceId,
+                                      String instanceId, // TODO Remove this parameter (unused/unnecessary)
                                       String attestationData,
                                       boolean requestServiceToken,
                                       Pkcs10Csr csr);
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java
index eedb7a2cc07..b99001476ea 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java
@@ -1,24 +1,29 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.athenz.identityprovider.client;
 
 import com.yahoo.container.core.identity.IdentityConfig;
 import com.yahoo.vespa.athenz.api.AthenzService;
+import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient;
+import com.yahoo.vespa.athenz.client.zts.InstanceIdentity;
+import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
 import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper;
 import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient;
 import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument;
+import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
 import com.yahoo.vespa.athenz.tls.KeyAlgorithm;
 import com.yahoo.vespa.athenz.tls.KeyUtils;
 import com.yahoo.vespa.athenz.tls.Pkcs10Csr;
-import com.yahoo.vespa.athenz.tls.Pkcs10CsrUtils;
 import com.yahoo.vespa.athenz.tls.SslContextBuilder;
 
 import javax.net.ssl.SSLContext;
 import java.io.File;
+import java.net.URI;
 import java.security.KeyPair;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
 
 import static com.yahoo.vespa.athenz.tls.KeyStoreType.JKS;
+import static java.util.Collections.singleton;
 
 /**
  * A service that provides method for initially registering the instance and refreshing it.
@@ -27,68 +32,74 @@ import static com.yahoo.vespa.athenz.tls.KeyStoreType.JKS;
  */
 class AthenzCredentialsService {
     private final IdentityConfig identityConfig;
-    private final IdentityDocumentClient identityDocumentClient;
-    private final ZtsClient ztsClient;
+    private final ServiceIdentityProvider nodeIdentityProvider;
     private final File trustStoreJks;
     private final String hostname;
 
     AthenzCredentialsService(IdentityConfig identityConfig,
-                             IdentityDocumentClient identityDocumentClient,
-                             ZtsClient ztsClient,
+                             ServiceIdentityProvider nodeIdentityProvider,
                              File trustStoreJks,
                              String hostname) {
         this.identityConfig = identityConfig;
-        this.identityDocumentClient = identityDocumentClient;
-        this.ztsClient = ztsClient;
+        this.nodeIdentityProvider = nodeIdentityProvider;
         this.trustStoreJks = trustStoreJks;
         this.hostname = hostname;
     }
 
     AthenzCredentials registerInstance() {
         KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA);
+        IdentityDocumentClient identityDocumentClient = createIdentityDocumentClient(identityConfig, nodeIdentityProvider);
         SignedIdentityDocument document = identityDocumentClient.getTenantIdentityDocument(hostname);
         InstanceCsrGenerator instanceCsrGenerator = new InstanceCsrGenerator(document.dnsSuffix());
+        AthenzService tenantIdentity = new AthenzService(identityConfig.domain(), identityConfig.service());
         Pkcs10Csr csr = instanceCsrGenerator.generateCsr(
-                new AthenzService(identityConfig.domain(), identityConfig.service()),
+                tenantIdentity,
                 document.providerUniqueId(),
                 document.identityDocument().ipAddresses(),
                 keyPair);
-        InstanceRegisterInformation instanceRegisterInformation =
-                new InstanceRegisterInformation(document.providerService().getFullName(),
-                                                identityConfig.domain(),
-                                                identityConfig.service(),
-                                                EntityBindingsMapper.toAttestationData(document),
-                                                Pkcs10CsrUtils.toPem(csr));
-        InstanceIdentity instanceIdentity = ztsClient.sendInstanceRegisterRequest(instanceRegisterInformation,
-                                                                                  document.ztsEndpoint());
-        return toAthenzCredentials(instanceIdentity, keyPair, document);
+
+        try (com.yahoo.vespa.athenz.client.zts.ZtsClient ztsClient =
+                     new DefaultZtsClient(URI.create(identityConfig.ztsUrl()), nodeIdentityProvider)) {
+            InstanceIdentity instanceIdentity =
+                    ztsClient.registerInstance(
+                            new AthenzService(identityConfig.configserverIdentityName()),
+                            tenantIdentity,
+                            null,
+                            EntityBindingsMapper.toAttestationData(document),
+                            true,
+                            csr);
+            return toAthenzCredentials(instanceIdentity, keyPair, document);
+        }
     }
 
     AthenzCredentials updateCredentials(SignedIdentityDocument document, SSLContext sslContext) {
+        AthenzService tenantIdentity = new AthenzService(identityConfig.domain(), identityConfig.service());
         KeyPair newKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA);
         InstanceCsrGenerator instanceCsrGenerator = new InstanceCsrGenerator(document.dnsSuffix());
         Pkcs10Csr csr = instanceCsrGenerator.generateCsr(
-                new AthenzService(identityConfig.domain(), identityConfig.service()),
+                tenantIdentity,
                 document.providerUniqueId(),
                 document.identityDocument().ipAddresses(),
                 newKeyPair);
-        InstanceRefreshInformation refreshInfo = new InstanceRefreshInformation(Pkcs10CsrUtils.toPem(csr));
-        InstanceIdentity instanceIdentity =
-                ztsClient.sendInstanceRefreshRequest(document.providerService().getFullName(),
-                                                     identityConfig.domain(),
-                                                     identityConfig.service(),
-                                                     document.providerUniqueId().asDottedString(),
-                                                     refreshInfo,
-                                                     document.ztsEndpoint(),
-                                                     sslContext);
-        return toAthenzCredentials(instanceIdentity, newKeyPair, document);
+
+        try (com.yahoo.vespa.athenz.client.zts.ZtsClient ztsClient =
+                     new DefaultZtsClient(URI.create(identityConfig.ztsUrl()), tenantIdentity, sslContext)) {
+            InstanceIdentity instanceIdentity =
+                    ztsClient.refreshInstance(
+                            new AthenzService(identityConfig.configserverIdentityName()),
+                            tenantIdentity,
+                            document.providerUniqueId().asDottedString(),
+                            true,
+                            csr);
+            return toAthenzCredentials(instanceIdentity, newKeyPair, document);
+        }
     }
 
     private AthenzCredentials toAthenzCredentials(InstanceIdentity instanceIdentity,
                                                   KeyPair keyPair,
                                                   SignedIdentityDocument identityDocument) {
-        X509Certificate certificate = instanceIdentity.getX509Certificate();
-        String serviceToken = instanceIdentity.getServiceToken();
+        X509Certificate certificate = instanceIdentity.certificate();
+        String serviceToken = instanceIdentity.nToken().get().getRawToken();
         SSLContext identitySslContext = createIdentitySslContext(keyPair.getPrivate(), certificate);
         return new AthenzCredentials(serviceToken, certificate, keyPair, identityDocument, identitySslContext);
     }
@@ -100,4 +111,11 @@ class AthenzCredentialsService {
                 .build();
     }
 
+    private static DefaultIdentityDocumentClient createIdentityDocumentClient(IdentityConfig config,
+                                                                              ServiceIdentityProvider nodeIdentityProvider) {
+        return new DefaultIdentityDocumentClient(
+                URI.create(config.loadBalancerAddress()),
+                nodeIdentityProvider,
+                new AthenzIdentityVerifier(singleton(new AthenzService(config.configserverIdentityName()))));
+    }
 }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java
index 32eb32ed366..3dc883f347f 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java
@@ -17,7 +17,6 @@ import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
 import com.yahoo.vespa.athenz.identity.ServiceIdentityProviderListenerHelper;
 import com.yahoo.vespa.athenz.identity.SiaIdentityProvider;
-import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
 import com.yahoo.vespa.athenz.tls.KeyStoreType;
 import com.yahoo.vespa.athenz.tls.SslContextBuilder;
 import com.yahoo.vespa.athenz.utils.SiaUtils;
@@ -25,7 +24,6 @@ import com.yahoo.vespa.defaults.Defaults;
 
 import javax.net.ssl.SSLContext;
 import java.io.File;
-import java.net.URI;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
 import java.time.Clock;
@@ -36,8 +34,6 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
-import static java.util.Collections.singleton;
-
 /**
  * @author mortent
  * @author bjorncs
@@ -71,8 +67,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
         this(config,
              metric,
              new AthenzCredentialsService(config,
-                                          createIdentityDocumentClient(config),
-                                          new ZtsClient(),
+                                          createNodeIdentityProvider(config),
                                           getDefaultTrustStoreLocation(),
                                           Defaults.getDefaults().vespaHostname()),
              new ScheduledThreadPoolExecutor(1),
@@ -201,12 +196,9 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
         }
     }
 
-    private static DefaultIdentityDocumentClient createIdentityDocumentClient(IdentityConfig config) {
-        return new DefaultIdentityDocumentClient(
-                URI.create(config.loadBalancerAddress()),
-                new SiaIdentityProvider(
-                        new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, getDefaultTrustStoreLocation()),
-                new AthenzIdentityVerifier(singleton(new AthenzService(config.configserverIdentityName()))));
+    private static SiaIdentityProvider createNodeIdentityProvider(IdentityConfig config) {
+        return new SiaIdentityProvider(
+                new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, getDefaultTrustStoreLocation());
     }
 
     private static File getDefaultTrustStoreLocation() {
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceIdentity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceIdentity.java
deleted file mode 100644
index 48200599149..00000000000
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceIdentity.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.athenz.identityprovider.client;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.yahoo.vespa.athenz.tls.X509CertificateUtils;
-
-import java.io.IOException;
-import java.security.cert.X509Certificate;
-
-/**
- * Used for deserializing response from ZTS
- *
- * @author mortent
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class InstanceIdentity {
-    @JsonProperty("x509Certificate") private final X509Certificate x509Certificate;
-    @JsonProperty("serviceToken") private final String serviceToken;
-
-    public InstanceIdentity(@JsonProperty("x509Certificate") @JsonDeserialize(using = X509CertificateDeserializer.class)
-                                    X509Certificate x509Certificate,
-                            @JsonProperty("serviceToken") String serviceToken) {
-        this.x509Certificate = x509Certificate;
-        this.serviceToken = serviceToken;
-    }
-
-    public X509Certificate getX509Certificate() {
-        return x509Certificate;
-    }
-
-    public String getServiceToken() {
-        return serviceToken;
-    }
-
-    public static class X509CertificateDeserializer extends JsonDeserializer<X509Certificate> {
-        @Override
-        public X509Certificate deserialize(JsonParser parser, DeserializationContext context) throws IOException {
-            return X509CertificateUtils.fromPem(parser.getValueAsString());
-        }
-    }
-
-}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRefreshInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRefreshInformation.java
deleted file mode 100644
index dd35cb7e401..00000000000
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRefreshInformation.java
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.athenz.identityprovider.client;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author bjorncs
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class InstanceRefreshInformation {
-
-    @JsonProperty("csr")
-    private final String csr;
-    @JsonProperty("token")
-    private final boolean requestServiceToken = true;
-
-    public InstanceRefreshInformation(String csr) {
-        this.csr = csr;
-    }
-}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRegisterInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRegisterInformation.java
deleted file mode 100644
index cdf47ad8624..00000000000
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRegisterInformation.java
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.athenz.identityprovider.client;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Used for serializing request to ZTS
- *
- * @author mortent
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class InstanceRegisterInformation {
-    @JsonProperty("provider")
-    private final String provider;
-    @JsonProperty("domain")
-    private final String domain;
-    @JsonProperty("service")
-    private final String service;
-    @JsonProperty("attestationData")
-    private final String attestationData;
-    @JsonProperty("ssh")
-    private final String ssh = null; // Not needed
-    @JsonProperty("csr")
-    private final String csr;
-    @JsonProperty("token")
-    private final boolean token = true;
-
-    public InstanceRegisterInformation(String provider, String domain, String service, String attestationData, String csr) {
-        this.provider = provider;
-        this.domain = domain;
-        this.service = service;
-        this.attestationData = attestationData;
-        this.csr = csr;
-    }
-}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java
index afdccac62cf..a3ec55eb815 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java
@@ -1,8 +1,6 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.athenz.identityprovider.client;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.athenz.zts.RoleCertificateRequest;
 import com.yahoo.athenz.zts.RoleToken;
 import com.yahoo.athenz.zts.ZTSClient;
@@ -10,22 +8,10 @@ import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzRole;
 import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.api.ZToken;
+import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient;
 import com.yahoo.vespa.athenz.tls.X509CertificateUtils;
-import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.methods.RequestBuilder;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.util.EntityUtils;
-import org.eclipse.jetty.http.HttpStatus;
 
 import javax.net.ssl.SSLContext;
-import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.net.URI;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
@@ -34,54 +20,11 @@ import java.time.Duration;
 /**
  * @author mortent
  * @author bjorncs
+ * @deprecated Will be replaced by {@link DefaultZtsClient} once role token/certificate caching is ready.
  */
+@Deprecated
 class ZtsClient {
 
-    private static final String INSTANCE_API_PATH = "/zts/v1/instance";
-
-    private final ObjectMapper objectMapper = new ObjectMapper();
-    private final HttpRequestRetryHandler retryHandler = new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true);
-
-    /**
-     * Send instance register request to ZTS, get InstanceIdentity
-     */
-     InstanceIdentity sendInstanceRegisterRequest(InstanceRegisterInformation instanceRegisterInformation,
-                                                         URI uri) {
-        try(CloseableHttpClient client = HttpClientBuilder.create().setRetryHandler(retryHandler).build()) {
-            HttpUriRequest postRequest = RequestBuilder.post()
-                    .setUri(uri.resolve(INSTANCE_API_PATH))
-                    .setEntity(toJsonStringEntity(instanceRegisterInformation))
-                    .build();
-            return getInstanceIdentity(client, postRequest);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
-    InstanceIdentity sendInstanceRefreshRequest(String providerService,
-                                                String instanceDomain,
-                                                String instanceServiceName,
-                                                String instanceId,
-                                                InstanceRefreshInformation instanceRefreshInformation,
-                                                URI ztsEndpoint,
-                                                SSLContext sslContext) {
-        try (CloseableHttpClient client = createHttpClientWithTlsAuth(sslContext, retryHandler)) {
-            URI uri = ztsEndpoint
-                    .resolve(INSTANCE_API_PATH + '/')
-                    .resolve(providerService + '/')
-                    .resolve(instanceDomain + '/')
-                    .resolve(instanceServiceName + '/')
-                    .resolve(instanceId);
-            HttpUriRequest postRequest = RequestBuilder.post()
-                    .setUri(uri)
-                    .setEntity(toJsonStringEntity(instanceRefreshInformation))
-                    .build();
-            return getInstanceIdentity(client, postRequest);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
     ZToken getRoleToken(AthenzDomain domain,
                         URI ztsEndpoint,
                         SSLContext sslContext) {
@@ -118,28 +61,4 @@ class ZtsClient {
         return X509CertificateUtils.fromPem(pemCert.token);
     }
 
-    private InstanceIdentity getInstanceIdentity(CloseableHttpClient client, HttpUriRequest postRequest)
-            throws IOException {
-        try (CloseableHttpResponse response = client.execute(postRequest)) {
-            if(HttpStatus.isSuccess(response.getStatusLine().getStatusCode())) {
-                return objectMapper.readValue(response.getEntity().getContent(), InstanceIdentity.class);
-            } else {
-                String message = EntityUtils.toString(response.getEntity());
-                throw new RuntimeException(String.format("Unable to get identity. http code/message: %d/%s",
-                                                         response.getStatusLine().getStatusCode(), message));
-            }
-        }
-    }
-
-    private StringEntity toJsonStringEntity(Object value) throws JsonProcessingException {
-        return new StringEntity(objectMapper.writeValueAsString(value), ContentType.APPLICATION_JSON);
-    }
-
-    private static CloseableHttpClient createHttpClientWithTlsAuth(SSLContext sslContext,
-                                                                   HttpRequestRetryHandler retryHandler) {
-            return HttpClientBuilder.create()
-                    .setRetryHandler(retryHandler)
-                    .setSSLContext(sslContext)
-                    .build();
-    }
 }