Merge pull request #4921 from vespa-engine/freva/config-server-cert-metrics

Report config server cert expiry metrics

3 files changed, 99 insertions, 9 deletions

diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java
index 706f797cd2c..beff50b52c6 100644
--- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java
@@ -19,6 +19,7 @@ import java.security.cert.Certificate;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -37,14 +38,15 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
     // TODO Make expiry and update frequency configurable parameters
     private static final Duration CERTIFICATE_EXPIRY_TIME = Duration.ofDays(30);
     private static final Duration CERTIFICATE_UPDATE_PERIOD = Duration.ofDays(7);
-    private static final String DUMMY_PASSWORD = "athenz";
+    private static final String CERTIFICATE_ALIAS = "athenz";
+    private static final String CERTIFICATE_PASSWORD = "athenz";
 
     private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
     private final AthenzCertificateClient certificateClient;
     private final KeyProvider keyProvider;
     private final AthenzProviderServiceConfig.Zones zoneConfig;
     private final AtomicBoolean alreadyConfigured = new AtomicBoolean();
-    private KeyStore initialKeyStore;
+    private volatile KeyStore currentKeyStore;
 
     @Inject
     public AthenzSslKeyStoreConfigurator(KeyProvider keyProvider,
@@ -54,7 +56,7 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
         this.certificateClient = new AthenzCertificateClient(config, zoneConfig);
         this.keyProvider = keyProvider;
         this.zoneConfig = zoneConfig;
-        this.initialKeyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig);
+        this.currentKeyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig);
     }
 
     @Override
@@ -62,8 +64,7 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
         if (alreadyConfigured.getAndSet(true)) { // For debugging purpose of SslKeyStoreConfigurator interface
             throw new IllegalStateException("Already configured. configure() can only be called once.");
         }
-        sslKeyStoreContext.updateKeyStore(initialKeyStore, DUMMY_PASSWORD);
-        initialKeyStore = null;
+        sslKeyStoreContext.updateKeyStore(currentKeyStore, CERTIFICATE_PASSWORD);
         scheduler.scheduleAtFixedRate(new AthenzCertificateUpdater(sslKeyStoreContext),
                                       CERTIFICATE_UPDATE_PERIOD.toMinutes()/*initial delay*/,
                                       CERTIFICATE_UPDATE_PERIOD.toMinutes(),
@@ -80,6 +81,12 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
         }
     }
 
+    Instant getKeyStoreExpiry() throws KeyStoreException {
+        X509Certificate certificate = (X509Certificate) currentKeyStore.getCertificate(CERTIFICATE_ALIAS);
+        return certificate.getNotAfter().toInstant();
+    }
+
+
     private static KeyStore downloadCertificate(KeyProvider keyProvider,
                                                 AthenzCertificateClient certificateClient,
                                                 AthenzProviderServiceConfig.Zones zoneConfig) {
@@ -90,7 +97,8 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
 
             KeyStore keyStore = KeyStore.getInstance("JKS");
             keyStore.load(null);
-            keyStore.setKeyEntry("athenz", privateKey, DUMMY_PASSWORD.toCharArray(), new Certificate[]{certificate});
+            keyStore.setKeyEntry(
+                    CERTIFICATE_ALIAS, privateKey, CERTIFICATE_PASSWORD.toCharArray(), new Certificate[]{certificate});
             return keyStore;
         } catch (IOException | NoSuchAlgorithmException | CertificateException | KeyStoreException e) {
             throw new RuntimeException(e);
@@ -118,8 +126,8 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
         public void run() {
             try {
                 log.log(LogLevel.INFO, "Updating Athenz certificate from ZTS");
-                KeyStore keyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig);
-                sslKeyStoreContext.updateKeyStore(keyStore, DUMMY_PASSWORD);
+                currentKeyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig);
+                sslKeyStoreContext.updateKeyStore(currentKeyStore, CERTIFICATE_PASSWORD);
                 log.log(LogLevel.INFO, "Athenz certificate reload successfully completed");
             } catch (Throwable e) {
                 log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + e.getMessage(), e);
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java
index 8c8b5de2a30..7e24109a197 100644
--- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java
@@ -25,6 +25,7 @@ import java.io.IOException;
 import java.math.BigInteger;
 import java.security.KeyPair;
 import java.security.KeyStore;
+import java.security.KeyStoreException;
 import java.security.Provider;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
@@ -39,6 +40,7 @@ import java.util.logging.Logger;
 public class AthenzSslTrustStoreConfigurator implements SslTrustStoreConfigurator {
 
     private static final Logger log = Logger.getLogger(AthenzSslTrustStoreConfigurator.class.getName());
+    private static final String CERTIFICATE_ALIAS = "cfgselfsigned";
 
     private static final Provider provider = new BouncyCastleProvider();
     private final KeyStore trustStore;
@@ -56,6 +58,11 @@ public class AthenzSslTrustStoreConfigurator implements SslTrustStoreConfigurato
         log.log(LogLevel.INFO, "Configured JDisc trust store with self-signed certificate");
     }
 
+    Instant getTrustStoreExpiry() throws KeyStoreException {
+        X509Certificate certificate = (X509Certificate) trustStore.getCertificate(CERTIFICATE_ALIAS);
+        return certificate.getNotAfter().toInstant();
+    }
+
     private static KeyStore createTrustStore(KeyProvider keyProvider,
                                              ConfigserverConfig configserverConfig,
                                              AthenzProviderServiceConfig athenzProviderServiceConfig) {
@@ -67,7 +74,7 @@ public class AthenzSslTrustStoreConfigurator implements SslTrustStoreConfigurato
             try (FileInputStream in = new FileInputStream(athenzProviderServiceConfig.athenzCaTrustStore())) {
                 trustStore.load(in, "changeit".toCharArray());
             }
-            trustStore.setCertificateEntry("cfgselfsigned", selfSignedCertificate);
+            trustStore.setCertificateEntry(CERTIFICATE_ALIAS, selfSignedCertificate);
             return trustStore;
         } catch (Exception e) {
             throw new RuntimeException(e);
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java
new file mode 100644
index 00000000000..cf734facf34
--- /dev/null
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java
@@ -0,0 +1,75 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.athenz.instanceproviderservice;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.jdisc.Metric;
+
+import com.google.inject.Inject;
+
+import java.security.KeyStoreException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author freva
+ */
+public class CertificateExpiryMetricUpdater extends AbstractComponent {
+
+    private static final Duration METRIC_REFRESH_PERIOD = Duration.ofMinutes(5);
+    private static final String NODE_CA_CERT_METRIC_NAME = "node-ca-cert.expiry.seconds";
+    private static final String ATHENZ_CONFIGSERVER_CERT_METRIC_NAME = "athenz-configserver-cert.expiry.seconds";
+
+    private final Logger logger = Logger.getLogger(CertificateExpiryMetricUpdater.class.getName());
+    private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
+    private final Metric metric;
+    private final AthenzSslKeyStoreConfigurator keyStoreConfigurator;
+    private final AthenzSslTrustStoreConfigurator trustStoreConfigurator;
+
+    @Inject
+    public CertificateExpiryMetricUpdater(Metric metric,
+                                          AthenzSslKeyStoreConfigurator keyStoreConfigurator,
+                                          AthenzSslTrustStoreConfigurator trustStoreConfigurator) {
+        this.metric = metric;
+        this.keyStoreConfigurator = keyStoreConfigurator;
+        this.trustStoreConfigurator = trustStoreConfigurator;
+
+
+        scheduler.scheduleAtFixedRate(this::updateMetrics,
+                30/*initial delay*/,
+                METRIC_REFRESH_PERIOD.getSeconds(),
+                TimeUnit.SECONDS);
+    }
+
+    @Override
+    public void deconstruct() {
+        try {
+            scheduler.shutdownNow();
+            scheduler.awaitTermination(30, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Failed to shutdown certificate expiry metrics updater on time", e);
+        }
+    }
+
+    private void updateMetrics() {
+        Instant now = Instant.now();
+
+        try {
+            Duration keyStoreExpiry = Duration.between(now, keyStoreConfigurator.getKeyStoreExpiry());
+            metric.set(ATHENZ_CONFIGSERVER_CERT_METRIC_NAME, keyStoreExpiry.getSeconds(), null);
+        } catch (KeyStoreException e) {
+            logger.log(Level.WARNING, "Failed to update key store expiry metric", e);
+        }
+
+        try {
+            Duration trustStoreExpiry = Duration.between(now, trustStoreConfigurator.getTrustStoreExpiry());
+            metric.set(NODE_CA_CERT_METRIC_NAME, trustStoreExpiry.getSeconds(), null);
+        } catch (KeyStoreException e) {
+            logger.log(Level.WARNING, "Failed to update trust store expiry metric", e);
+        }
+    }
+}