Added ConfigServerKeyStoreRefresher

4 files changed, 371 insertions, 6 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
index 7376a59bd5c..5581415fec2 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
@@ -86,11 +86,13 @@ public class Environment {
                 createKeyStoreOptions(
                         configServerConfig.keyStoreConfig().path(),
                         configServerConfig.keyStoreConfig().password().toCharArray(),
-                        configServerConfig.keyStoreConfig().type().name()),
+                        configServerConfig.keyStoreConfig().type().name(),
+                        "BC"),
                 createKeyStoreOptions(
                         configServerConfig.trustStoreConfig().path(),
                         configServerConfig.trustStoreConfig().password().toCharArray(),
-                        configServerConfig.trustStoreConfig().type().name()),
+                        configServerConfig.trustStoreConfig().type().name(),
+                        null),
                 createAthenzIdentity(
                         configServerConfig.athenzDomain(),
                         configServerConfig.serviceName())
@@ -161,10 +163,10 @@ public class Environment {
         return Arrays.asList(logstashNodes.split("[,\\s]+"));
     }
 
-    private static Optional<KeyStoreOptions> createKeyStoreOptions(String pathToKeyStore, char[] password, String type) {
+    private static Optional<KeyStoreOptions> createKeyStoreOptions(String pathToKeyStore, char[] password, String type, String provider) {
         return Optional.ofNullable(pathToKeyStore)
                 .filter(path -> !Strings.isNullOrEmpty(path))
-                .map(path -> new KeyStoreOptions(Paths.get(path), password, type));
+                .map(path -> new KeyStoreOptions(Paths.get(path), password, type, provider));
     }
 
     private static Optional<AthenzIdentity> createAthenzIdentity(String athenzDomain, String serviceName) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresher.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresher.java
new file mode 100644
index 00000000000..1595db1047a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresher.java
@@ -0,0 +1,188 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.certificate;
+
+import com.yahoo.net.HostName;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.Clock;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Automatically refreshes the KeyStore used to authenticate this node to the configserver.
+ * The keystore contains a single certificate signed by one of the configservers.
+ *
+ * @author freva
+ */
+public class ConfigServerKeyStoreRefresher {
+
+    private static final Logger logger = Logger.getLogger(ConfigServerKeyStoreRefresher.class.getName());
+    private static final String KEY_STORE_ALIAS = "alias";
+    static final long MINIMUM_SECONDS_BETWEEN_REFRESH_RETRY = 3600;
+    static final String SIGNER_ALGORITHM = "SHA256withRSA";
+    static final String CONFIG_SERVER_CERTIFICATE_SIGNING_PATH = "/athenz/v1/provider/sign";
+
+    private final ScheduledExecutorService executor;
+    private final KeyStoreOptions keyStoreOptions;
+    private final Runnable keyStoreUpdatedCallback;
+    private final ConfigServerApi configServerApi;
+    private final Clock clock;
+    private final String hostname;
+
+    public ConfigServerKeyStoreRefresher(
+            KeyStoreOptions keyStoreOptions, Runnable keyStoreUpdatedCallback, ConfigServerApi configServerApi) {
+        this(keyStoreOptions, keyStoreUpdatedCallback, configServerApi, Executors.newScheduledThreadPool(0),
+                Clock.systemUTC(), HostName.getLocalhost());
+    }
+
+    ConfigServerKeyStoreRefresher(KeyStoreOptions keyStoreOptions,
+                                  Runnable keyStoreUpdatedCallback,
+                                  ConfigServerApi configServerApi,
+                                  ScheduledExecutorService executor,
+                                  Clock clock,
+                                  String hostname) {
+        this.keyStoreOptions = keyStoreOptions;
+        this.keyStoreUpdatedCallback = keyStoreUpdatedCallback;
+        this.configServerApi = configServerApi;
+        this.executor = executor;
+        this.clock = clock;
+        this.hostname = hostname;
+    }
+
+    public void start() {
+        executor.schedule(this::refresh, getSecondsUntilNextRefresh(), TimeUnit.SECONDS);
+    }
+
+    void refresh() {
+        try {
+            if (refreshKeyStoreIfNeeded()) {
+                keyStoreUpdatedCallback.run();
+            }
+            final long secondsUntilNextRefresh = getSecondsUntilNextRefresh();
+            executor.schedule(this::refresh, secondsUntilNextRefresh, TimeUnit.SECONDS);
+            logger.log(Level.INFO, "Successfully updated keystore, scheduled next refresh in " +
+                    secondsUntilNextRefresh + "sec");
+        } catch (Exception e) {
+            logger.log(Level.WARNING, "Failed to update keystore on schedule, will try again in " +
+                    MINIMUM_SECONDS_BETWEEN_REFRESH_RETRY + "sec", e);
+            executor.schedule(this::refresh, MINIMUM_SECONDS_BETWEEN_REFRESH_RETRY, TimeUnit.SECONDS);
+        }
+    }
+
+    public void stop() {
+        do {
+            try {
+                executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+            } catch (InterruptedException e1) {
+                logger.info("Interrupted while waiting for ConfigServerKeyStoreRefresher thread to shutdown");
+            }
+        } while (!executor.isTerminated());
+    }
+
+    public boolean refreshKeyStoreIfNeeded() throws
+            IOException, NoSuchAlgorithmException, OperatorCreationException, CertificateException, KeyStoreException, NoSuchProviderException {
+        if (!shouldRefreshCertificate()) return false;
+
+        KeyPair keyPair = generateKeyPair();
+        PKCS10CertificationRequest csr = generateCsr(keyPair, hostname);
+        X509Certificate certificate = sendCsr(csr);
+
+        storeCertificate(keyPair, certificate);
+        return true;
+    }
+
+    private long getSecondsUntilNextRefresh() {
+        long secondsUntilNextCheck = 0;
+        try {
+            secondsUntilNextCheck = getSecondsUntilCertificateShouldBeRefreshed();
+        } catch (Exception e) {
+            logger.log(Level.WARNING, "Failed to get remaining certificate lifetime", e);
+        }
+
+        return Math.max(MINIMUM_SECONDS_BETWEEN_REFRESH_RETRY, secondsUntilNextCheck);
+    }
+
+    private boolean shouldRefreshCertificate() {
+        try {
+            return getSecondsUntilCertificateShouldBeRefreshed() <= 0;
+        } catch (Exception e) { // We can't read the key store for whatever reason, let's just try to refresh it
+            return true;
+        }
+    }
+
+    /**
+     * Returns number of seconds until we should start trying to refresh the certificate, this should be
+     * well before the certificate actually expires so that we have enough time to retry without
+     * overloading config server.
+     */
+    private long getSecondsUntilCertificateShouldBeRefreshed()
+            throws NoSuchAlgorithmException, CertificateException, NoSuchProviderException, KeyStoreException, IOException {
+        X509Certificate cert = getConfigServerCertificate();
+        long notBefore = cert.getNotBefore().getTime() / 1000;
+        long notAfter = cert.getNotAfter().getTime() / 1000;
+        long now = clock.millis() / 1000;
+        long thirdOfLifetime = (notAfter - notBefore) / 3;
+
+        return Math.max(0, notBefore + thirdOfLifetime - now);
+    }
+
+    X509Certificate getConfigServerCertificate() throws NoSuchAlgorithmException, CertificateException, NoSuchProviderException, KeyStoreException, IOException {
+        return (X509Certificate) keyStoreOptions.loadKeyStore().getCertificate(KEY_STORE_ALIAS);
+    }
+
+    private void storeCertificate(KeyPair keyPair, X509Certificate certificate)
+            throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException, NoSuchProviderException {
+        keyStoreOptions.path.getParent().toFile().mkdirs();
+        X509Certificate[] certificateChain = {certificate};
+
+        try (FileOutputStream fos = new FileOutputStream(keyStoreOptions.path.toFile())) {
+            KeyStore keyStore = keyStoreOptions.getKeyStoreInstance();
+            keyStore.load(null, null);
+            keyStore.setKeyEntry(KEY_STORE_ALIAS, keyPair.getPrivate(), keyStoreOptions.password, certificateChain);
+            keyStore.store(fos, keyStoreOptions.password);
+        }
+    }
+
+    private X509Certificate sendCsr(PKCS10CertificationRequest csr) {
+        CertificateSerializedPayload certificateSerializedPayload = configServerApi.post(
+                CONFIG_SERVER_CERTIFICATE_SIGNING_PATH,
+                new CsrSerializedPayload(csr),
+                CertificateSerializedPayload.class);
+
+        return certificateSerializedPayload.certificate;
+    }
+
+    static KeyPair generateKeyPair() throws NoSuchAlgorithmException {
+        KeyPairGenerator rsa = KeyPairGenerator.getInstance("RSA");
+        rsa.initialize(2048);
+        return rsa.genKeyPair();
+    }
+
+    private static PKCS10CertificationRequest generateCsr(KeyPair keyPair, String commonName)
+            throws NoSuchAlgorithmException, OperatorCreationException {
+        ContentSigner signer = new JcaContentSignerBuilder(SIGNER_ALGORITHM).build(keyPair.getPrivate());
+
+        return new JcaPKCS10CertificationRequestBuilder(new X500Name("CN=" + commonName), keyPair.getPublic())
+                .build(signer);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java
index dc3c3e754c3..1115f6dca91 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java
@@ -9,24 +9,37 @@ import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.security.NoSuchProviderException;
 import java.security.cert.CertificateException;
+import java.util.Optional;
 
 public class KeyStoreOptions {
     public final Path path;
     public final char[] password;
     public final String type;
+    private final Optional<String> provider;
 
     public KeyStoreOptions(Path path, char[] password, String type) {
+        this(path, password, type, null);
+    }
+
+    public KeyStoreOptions(Path path, char[] password, String type, String provider) {
         this.path = path;
         this.password = password;
         this.type = type;
+        this.provider = Optional.ofNullable(provider);
     }
 
-    public KeyStore loadKeyStoreWithBcProvider()
+    public KeyStore loadKeyStore()
             throws IOException, NoSuchProviderException, KeyStoreException, CertificateException, NoSuchAlgorithmException {
         try (FileInputStream in = new FileInputStream(path.toFile())) {
-            KeyStore keyStore = KeyStore.getInstance(type, "BC");
+            KeyStore keyStore = getKeyStoreInstance();
             keyStore.load(in, password);
             return keyStore;
         }
     }
+
+    public KeyStore getKeyStoreInstance() throws NoSuchProviderException, KeyStoreException {
+        return provider.isPresent() ?
+                KeyStore.getInstance(type, provider.get()) :
+                KeyStore.getInstance(type);
+    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherTest.java
new file mode 100644
index 00000000000..f9f8b230154
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherTest.java
@@ -0,0 +1,162 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.certificate;
+
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.util.Date;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author freva
+ */
+public class ConfigServerKeyStoreRefresherTest {
+
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
+    private final ManualClock clock = new ManualClock();
+    private final String commonName = "CertificateRefresherTest";
+    private final Duration certificateExpiration = Duration.ofDays(6);
+    private final ConfigServerApi configServerApi = mock(ConfigServerApi.class);
+    private final Runnable keyStoreUpdatedCallback = mock(Runnable.class);
+    private final ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
+    private KeyStoreOptions keyStoreOptions;
+
+    @Before
+    public void setup() {
+        keyStoreOptions = new KeyStoreOptions(
+                tempFolder.getRoot().toPath().resolve("some/path/keystore.p12"), new char[0], "PKCS12", null);
+    }
+
+    @Test
+    public void manually_trigger_certificate_refresh() throws Exception {
+        X509Certificate firstCertificate = mockConfigServerCertificateSigning(1);
+
+        ConfigServerKeyStoreRefresher keyStoreRefresher = new ConfigServerKeyStoreRefresher(
+                keyStoreOptions, keyStoreUpdatedCallback, configServerApi, executor, clock, commonName);
+
+        // No keystore previously existed, so a new one should be written
+        assertTrue(keyStoreRefresher.refreshKeyStoreIfNeeded());
+        assertEquals(firstCertificate, keyStoreRefresher.getConfigServerCertificate());
+
+        // Calling it again before a third of certificate lifetime has passed has no effect
+        assertFalse(keyStoreRefresher.refreshKeyStoreIfNeeded());
+        assertEquals(firstCertificate, keyStoreRefresher.getConfigServerCertificate());
+
+        // After a third of the expiration time passes, we should refresh the certificate
+        clock.advance(certificateExpiration.dividedBy(3).plusSeconds(1));
+        X509Certificate secondCertificate = mockConfigServerCertificateSigning(2);
+        assertTrue(keyStoreRefresher.refreshKeyStoreIfNeeded());
+        assertEquals(secondCertificate, keyStoreRefresher.getConfigServerCertificate());
+
+        verify(configServerApi, times(2))
+                .post(eq(ConfigServerKeyStoreRefresher.CONFIG_SERVER_CERTIFICATE_SIGNING_PATH), any(), any());
+
+        // We're just triggering refresh manually, so callback and executor should not have been touched
+        verifyZeroInteractions(keyStoreUpdatedCallback);
+        verifyZeroInteractions(executor);
+    }
+
+    @Test
+    public void certificate_refresh_schedule_test() throws Exception {
+        ConfigServerKeyStoreRefresher keyStoreRefresher = new ConfigServerKeyStoreRefresher(
+                keyStoreOptions, keyStoreUpdatedCallback, configServerApi, executor, clock, commonName);
+
+        // No keystore exist, so refresh once
+        mockConfigServerCertificateSigning(1);
+        assertTrue(keyStoreRefresher.refreshKeyStoreIfNeeded());
+
+        // Start automatic refreshment, since keystore was just written, next check should be in 1/3rd of
+        // certificate lifetime, which is in 2 days.
+        keyStoreRefresher.start();
+        Duration nextExpectedExecution = Duration.ofDays(2);
+        verify(executor, times(1)).schedule(any(Runnable.class), eq(nextExpectedExecution.getSeconds()), eq(TimeUnit.SECONDS));
+
+        // First automatic refreshment goes without any problems
+        clock.advance(nextExpectedExecution);
+        mockConfigServerCertificateSigning(2);
+        keyStoreRefresher.refresh();
+        verify(executor, times(2)).schedule(any(Runnable.class), eq(nextExpectedExecution.getSeconds()), eq(TimeUnit.SECONDS));
+        verify(keyStoreUpdatedCallback).run();
+
+        // We fail to refresh the certificate, wait minimum amount of time and try again
+        clock.advance(nextExpectedExecution);
+        mockConfigServerCertificateSigningFailure(new RuntimeException());
+        keyStoreRefresher.refresh();
+        nextExpectedExecution = Duration.ofSeconds(ConfigServerKeyStoreRefresher.MINIMUM_SECONDS_BETWEEN_REFRESH_RETRY);
+        verify(executor, times(1)).schedule(any(Runnable.class), eq(nextExpectedExecution.getSeconds()), eq(TimeUnit.SECONDS));
+
+        clock.advance(nextExpectedExecution);
+        keyStoreRefresher.refresh();
+        verify(executor, times(2)).schedule(any(Runnable.class), eq(nextExpectedExecution.getSeconds()), eq(TimeUnit.SECONDS));
+        verifyNoMoreInteractions(keyStoreUpdatedCallback); // Callback not called after the last 2 failures
+
+        clock.advance(nextExpectedExecution);
+        mockConfigServerCertificateSigning(3);
+        keyStoreRefresher.refresh();
+        nextExpectedExecution = Duration.ofDays(2);
+        verify(executor, times(3)).schedule(any(Runnable.class), eq(nextExpectedExecution.getSeconds()), eq(TimeUnit.SECONDS));
+        verify(keyStoreUpdatedCallback, times(2)).run();
+    }
+
+    private X509Certificate mockConfigServerCertificateSigning(int serial) throws Exception {
+        X509Certificate certificate = makeCertificate(serial);
+
+        when(configServerApi.post(eq(ConfigServerKeyStoreRefresher.CONFIG_SERVER_CERTIFICATE_SIGNING_PATH), any(), any()))
+                .thenReturn(new CertificateSerializedPayload(certificate));
+        return certificate;
+    }
+
+    private void mockConfigServerCertificateSigningFailure(Exception exception) throws Exception {
+        when(configServerApi.post(eq(ConfigServerKeyStoreRefresher.CONFIG_SERVER_CERTIFICATE_SIGNING_PATH), any(), any()))
+                .thenThrow(exception);
+    }
+
+    private X509Certificate makeCertificate(int serial) throws Exception {
+        try {
+            KeyPair keyPair = ConfigServerKeyStoreRefresher.generateKeyPair();
+            X500Name subject = new X500Name("CN=" + commonName);
+            Date notBefore = Date.from(clock.instant());
+            Date notAfter = Date.from(clock.instant().plus(certificateExpiration));
+
+            JcaX509v3CertificateBuilder certGen = new JcaX509v3CertificateBuilder(subject,
+                    BigInteger.valueOf(serial), notBefore, notAfter, subject, keyPair.getPublic());
+            ContentSigner sigGen = new JcaContentSignerBuilder(ConfigServerKeyStoreRefresher.SIGNER_ALGORITHM)
+                    .build(keyPair.getPrivate());
+            return new JcaX509CertificateConverter()
+                    .setProvider(new BouncyCastleProvider())
+                    .getCertificate(certGen.build(sigGen));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
\ No newline at end of file