Added athenz-tenant-cert.expiry metric

2 files changed, 58 insertions, 7 deletions

diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImpl.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImpl.java
index 8828a345b53..c32d08c97ff 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImpl.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImpl.java
@@ -6,6 +6,7 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.container.core.identity.IdentityConfig;
 import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider;
 import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.log.LogLevel;
 
 import javax.net.ssl.KeyManager;
@@ -52,10 +53,13 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
     static final int BACKOFF_DELAY_MULTIPLIER = 2;
     static final Duration AWAIT_TERMINTATION_TIMEOUT = Duration.ofSeconds(90);
 
+    private static final Duration CERTIFICATE_EXPIRY_METRIC_UPDATE_PERIOD = Duration.ofMinutes(5);
+    private static final String CERTIFICATE_EXPIRY_METRIC_NAME = "athenz-tenant-cert.expiry.seconds";
 
     static final String REGISTER_INSTANCE_TAG = "register-instance";
     static final String UPDATE_CREDENTIALS_TAG = "update-credentials";
     static final String TIMEOUT_INITIAL_WAIT_TAG = "timeout-initial-wait";
+    static final String METRICS_UPDATER_TAG = "metrics-updater";
 
 
     private final AtomicReference<AthenzCredentials> credentials = new AtomicReference<>();
@@ -67,9 +71,12 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
     private final String domain;
     private final String service;
 
+    private final CertificateExpiryMetricUpdater metricUpdater;
+
     @Inject
-    public AthenzIdentityProviderImpl(IdentityConfig config) {
+    public AthenzIdentityProviderImpl(IdentityConfig config, Metric metric) {
         this(config,
+             metric,
              new AthenzCredentialsService(config,
                                           new IdentityDocumentService(config.loadBalancerAddress()),
                                           new AthenzService(),
@@ -80,6 +87,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
 
     // Test only
     AthenzIdentityProviderImpl(IdentityConfig config,
+                               Metric metric,
                                AthenzCredentialsService athenzCredentialsService,
                                Scheduler scheduler,
                                Clock clock) {
@@ -90,6 +98,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
         this.service = config.service();
         scheduler.submit(new RegisterInstanceTask());
         scheduler.schedule(new TimeoutInitialWaitTask(), INITIAL_WAIT_NTOKEN);
+
+        metricUpdater = new CertificateExpiryMetricUpdater(metric);
     }
 
     @Override
@@ -196,6 +206,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
                 credentials.set(athenzCredentialsService.registerInstance());
                 credentialsRetrievedSignal.countDown();
                 scheduler.schedule(new UpdateCredentialsTask(), UPDATE_PERIOD);
+                scheduler.submit(metricUpdater);
             } catch (Throwable t) {
                 log.log(LogLevel.ERROR, "Failed to register instance: " + t.getMessage(), t);
                 lastThrowable.set(t);
@@ -240,6 +251,27 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
         }
     }
 
+    private class CertificateExpiryMetricUpdater implements RunnableWithTag {
+        private final Metric metric;
+
+        private CertificateExpiryMetricUpdater(Metric metric) {
+            this.metric = metric;
+        }
+
+        @Override
+        public void run() {
+            Instant expirationTime = getExpirationTime(credentials.get());
+            Duration remainingLifetime = Duration.between(clock.instant(), expirationTime);
+            metric.set(CERTIFICATE_EXPIRY_METRIC_NAME, remainingLifetime.getSeconds(), null);
+            scheduler.schedule(this, CERTIFICATE_EXPIRY_METRIC_UPDATE_PERIOD);
+        }
+
+        @Override
+        public String tag() {
+            return METRICS_UPDATER_TAG;
+        }
+    }
+
     private class TimeoutInitialWaitTask implements RunnableWithTag {
         @Override
         public void run() {
diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImplTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImplTest.java
index 1c0efef2089..1ee23334a16 100644
--- a/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImplTest.java
+++ b/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImplTest.java
@@ -5,6 +5,7 @@ import com.yahoo.container.core.identity.IdentityConfig;
 import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider;
 import com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.RunnableWithTag;
 import com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.Scheduler;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.test.ManualClock;
 import org.junit.Test;
 
@@ -13,14 +14,19 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.PriorityQueue;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.INITIAL_BACKOFF_DELAY;
 import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.INITIAL_WAIT_NTOKEN;
 import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.MAX_REGISTER_BACKOFF_DELAY;
+import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.METRICS_UPDATER_TAG;
 import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.REDUCED_UPDATE_PERIOD;
 import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.REGISTER_INSTANCE_TAG;
 import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.TIMEOUT_INITIAL_WAIT_TAG;
@@ -38,10 +44,22 @@ import static org.mockito.Mockito.when;
  */
 public class AthenzIdentityProviderImplTest {
 
+    private static final Metric DUMMY_METRIC = new Metric() {
+        @Override
+        public void set(String s, Number number, Context context) {}
+        @Override
+        public void add(String s, Number number, Context context) {}
+        @Override
+        public Context createContext(Map<String, ?> stringMap) { return null; }
+    };
+
     private static final IdentityConfig IDENTITY_CONFIG =
             new IdentityConfig(new IdentityConfig.Builder()
                                        .service("tenantService").domain("tenantDomain").loadBalancerAddress("cfg"));
 
+    private final Set<String> IGNORED_TASKS = Stream.of(UPDATE_CREDENTIALS_TAG, METRICS_UPDATER_TAG)
+            .collect(Collectors.toSet());
+
     @Test
     public void athenz_credentials_are_retrieved_after_component_contruction_completed() {
         IdentityDocumentService identityDocumentService = mock(IdentityDocumentService.class);
@@ -56,7 +74,7 @@ public class AthenzIdentityProviderImplTest {
                 new AthenzCredentialsService(IDENTITY_CONFIG, identityDocumentService, athenzService, clock);
 
         AthenzIdentityProvider identityProvider =
-                new AthenzIdentityProviderImpl(IDENTITY_CONFIG, credentialService, scheduler, clock);
+                new AthenzIdentityProviderImpl(IDENTITY_CONFIG, DUMMY_METRIC, credentialService, scheduler, clock);
 
         List<MockScheduler.CompletedTask> expectedTasks =
                 Arrays.asList(
@@ -64,7 +82,7 @@ public class AthenzIdentityProviderImplTest {
                         new MockScheduler.CompletedTask(TIMEOUT_INITIAL_WAIT_TAG, INITIAL_WAIT_NTOKEN));
         // Don't run update credential tasks, otherwise infinite loop
         List<MockScheduler.CompletedTask> completedTasks =
-                scheduler.runAllTasks(task -> !task.tag().equals(UPDATE_CREDENTIALS_TAG));
+                scheduler.runAllTasks(task -> !IGNORED_TASKS.contains(task.tag()));
         assertEquals(expectedTasks, completedTasks);
         assertEquals("TOKEN", identityProvider.getNToken());
     }
@@ -83,7 +101,7 @@ public class AthenzIdentityProviderImplTest {
         ManualClock clock = new ManualClock(Instant.EPOCH);
         MockScheduler scheduler = new MockScheduler(clock);
         AthenzIdentityProvider identityProvider =
-                new AthenzIdentityProviderImpl(IDENTITY_CONFIG, credentialService, scheduler, clock);
+                new AthenzIdentityProviderImpl(IDENTITY_CONFIG, DUMMY_METRIC, credentialService, scheduler, clock);
 
         List<MockScheduler.CompletedTask> expectedTasks =
                 Arrays.asList(
@@ -96,7 +114,7 @@ public class AthenzIdentityProviderImplTest {
                         new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, MAX_REGISTER_BACKOFF_DELAY));
         // Don't run update credential tasks, otherwise infinite loop
         List<MockScheduler.CompletedTask> completedTasks =
-                scheduler.runAllTasks(task -> !task.tag().equals(UPDATE_CREDENTIALS_TAG));
+                scheduler.runAllTasks(task -> !IGNORED_TASKS.contains(task.tag()));
         assertEquals(expectedTasks, completedTasks);
         assertEquals("TOKEN", identityProvider.getNToken());
     }
@@ -121,7 +139,7 @@ public class AthenzIdentityProviderImplTest {
                 new AthenzCredentialsService(IDENTITY_CONFIG, identityDocumentService, athenzService, clock);
 
         AthenzIdentityProvider identityProvider =
-                new AthenzIdentityProviderImpl(IDENTITY_CONFIG, credentialService, scheduler, clock);
+                new AthenzIdentityProviderImpl(IDENTITY_CONFIG, DUMMY_METRIC, credentialService, scheduler, clock);
 
         List<MockScheduler.CompletedTask> expectedTasks =
                 Arrays.asList(
@@ -134,7 +152,8 @@ public class AthenzIdentityProviderImplTest {
                         new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, UPDATE_PERIOD));
         AtomicInteger counter = new AtomicInteger(0);
         List<MockScheduler.CompletedTask> completedTasks =
-                scheduler.runAllTasks(task -> counter.getAndIncrement() < 7); // 1 registration + 1 timeout + 5 update tasks
+                scheduler.runAllTasks(task -> !task.tag().equals(METRICS_UPDATER_TAG) &&
+                        counter.getAndIncrement() < expectedTasks.size());
         assertEquals(expectedTasks, completedTasks);
         assertEquals("TOKEN", identityProvider.getNToken());
     }