Report metrics on Athenz client errors

5 files changed, 52 insertions, 14 deletions

diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java
index 3adc445a781..bf2a6719842 100644
--- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java
+++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java
@@ -224,7 +224,7 @@ public class CertificateAuthorityApiTest extends ContainerTester {
         private final X509Certificate certificate;
 
         public TestZtsClient(Principal principal, X509Certificate certificate, URI ztsUrl, SSLContext sslContext) {
-            super(ztsUrl, () -> sslContext, null);
+            super(ztsUrl, () -> sslContext, null, ErrorHandler.empty());
             this.principal = principal;
             this.certificate = certificate;
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java
index 1626e4fd559..9d8a5e277b4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.athenz.impl;
 
 import com.google.inject.Inject;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.client.zms.DefaultZmsClient;
 import com.yahoo.vespa.athenz.client.zms.ZmsClient;
@@ -10,21 +11,31 @@ import com.yahoo.vespa.athenz.client.zts.ZtsClient;
 import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory;
 import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig;
+import org.apache.http.client.methods.HttpUriRequest;
 
 import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * @author bjorncs
  */
 public class AthenzClientFactoryImpl implements AthenzClientFactory {
 
+    private static final String METRIC_NAME = "athenz.request.error";
+    private static final String ATHENZ_SERVICE_DIMENSION = "athenzService";
+
     private final AthenzConfig config;
     private final ServiceIdentityProvider identityProvider;
+    private final Metric metrics;
+    private final Map<String, Metric.Context> metricContexts;
 
     @Inject
-    public AthenzClientFactoryImpl(ServiceIdentityProvider identityProvider, AthenzConfig config) {
+    public AthenzClientFactoryImpl(ServiceIdentityProvider identityProvider, AthenzConfig config, Metric metrics) {
         this.identityProvider = identityProvider;
         this.config = config;
+        this.metrics = metrics;
+        this.metricContexts = new HashMap<>();
     }
 
     @Override
@@ -37,7 +48,7 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory {
      */
     @Override
     public ZmsClient createZmsClient() {
-        return new DefaultZmsClient(URI.create(config.zmsUrl()), identityProvider);
+        return new DefaultZmsClient(URI.create(config.zmsUrl()), identityProvider, this::reportMetricErrorHandler);
     }
 
     /**
@@ -53,4 +64,9 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory {
         return true;
     }
 
+    private void reportMetricErrorHandler(HttpUriRequest request, Exception error) {
+        String hostname = request.getURI().getHost();
+        Metric.Context context = metricContexts.computeIfAbsent(hostname, host -> metrics.createContext(Map.of(ATHENZ_SERVICE_DIMENSION, host)));
+        metrics.add(METRIC_NAME, 1, context);
+    }
 }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java
index c1ce45c35da..37ef513c786 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java
@@ -39,12 +39,15 @@ public abstract class ClientBase implements AutoCloseable {
 
     private final CloseableHttpClient client;
     private final ClientExceptionFactory exceptionFactory;
+    private final ErrorHandler errorHandler;
 
     protected ClientBase(String userAgent,
                          Supplier<SSLContext> sslContextSupplier,
                          ClientExceptionFactory exceptionFactory,
-                         HostnameVerifier hostnameVerifier) {
+                         HostnameVerifier hostnameVerifier,
+                         ErrorHandler errorHandler) {
         this.exceptionFactory = exceptionFactory;
+        this.errorHandler = errorHandler;
         this.client = createHttpClient(userAgent, sslContextSupplier, hostnameVerifier);
     }
 
@@ -52,10 +55,17 @@ public abstract class ClientBase implements AutoCloseable {
         try {
             return client.execute(request, responseHandler);
         } catch (IOException e) {
+            try {
+                reportError(request, e);
+            } catch (Exception _ignored) {}
             throw new UncheckedIOException(e);
         }
     }
 
+    private void reportError(HttpUriRequest request, Exception e) {
+        errorHandler.reportError(request, e);
+    }
+
     protected StringEntity toJsonStringEntity(Object entity) {
         try {
             return new StringEntity(objectMapper.writeValueAsString(entity), ContentType.APPLICATION_JSON);
@@ -114,4 +124,11 @@ public abstract class ClientBase implements AutoCloseable {
     protected interface ClientExceptionFactory {
         RuntimeException createException(int errorCode, String description);
     }
+
+    public interface ErrorHandler {
+        static ErrorHandler empty() {
+            return (r,e)->{};
+        }
+        void reportError(HttpUriRequest request, Exception error);
+    }
 }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java
index 33cb6d7d5d4..3742996c274 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java
@@ -25,7 +25,6 @@ import javax.net.ssl.SSLContext;
 import java.net.URI;
 import java.util.Collections;
 import java.util.List;
-import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.Set;
 import java.util.function.Supplier;
@@ -40,16 +39,16 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient {
     private final URI zmsUrl;
     private final AthenzIdentity identity;
 
-    public DefaultZmsClient(URI zmsUrl, AthenzIdentity identity, SSLContext sslContext) {
-        this(zmsUrl, identity, () -> sslContext);
+    public DefaultZmsClient(URI zmsUrl, AthenzIdentity identity, SSLContext sslContext, ErrorHandler errorHandler) {
+        this(zmsUrl, identity, () -> sslContext, errorHandler);
     }
 
-    public DefaultZmsClient(URI zmsUrl, ServiceIdentityProvider identityProvider) {
-        this(zmsUrl, identityProvider.identity(), identityProvider::getIdentitySslContext);
+    public DefaultZmsClient(URI zmsUrl, ServiceIdentityProvider identityProvider, ErrorHandler errorHandler) {
+        this(zmsUrl, identityProvider.identity(), identityProvider::getIdentitySslContext, errorHandler);
     }
 
-    private DefaultZmsClient(URI zmsUrl, AthenzIdentity identity, Supplier<SSLContext> sslContextSupplier) {
-        super("vespa-zms-client", sslContextSupplier, ZmsClientException::new, null);
+    private DefaultZmsClient(URI zmsUrl, AthenzIdentity identity, Supplier<SSLContext> sslContextSupplier, ErrorHandler errorHandler) {
+        super("vespa-zms-client", sslContextSupplier, ZmsClientException::new, null, errorHandler);
         this.zmsUrl = addTrailingSlash(zmsUrl);
         this.identity = identity;
     }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java
index e8c0072731a..28119dc1f5a 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java
@@ -53,8 +53,8 @@ import static java.util.stream.Collectors.toList;
 public class DefaultZtsClient extends ClientBase implements ZtsClient {
 
     private final URI ztsUrl;
-    protected DefaultZtsClient(URI ztsUrl, Supplier<SSLContext> sslContextSupplier, HostnameVerifier hostnameVerifier) {
-        super("vespa-zts-client", sslContextSupplier, ZtsClientException::new, hostnameVerifier);
+    protected DefaultZtsClient(URI ztsUrl, Supplier<SSLContext> sslContextSupplier, HostnameVerifier hostnameVerifier, ErrorHandler errorHandler) {
+        super("vespa-zts-client", sslContextSupplier, ZtsClientException::new, hostnameVerifier, errorHandler);
         this.ztsUrl = addTrailingSlash(ztsUrl);
     }
 
@@ -225,6 +225,7 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient {
     }
     public static class Builder {
         private URI ztsUrl;
+        private ClientBase.ErrorHandler errorHandler = ErrorHandler.empty();
         private HostnameVerifier hostnameVerifier = null;
         private Supplier<SSLContext> sslContextSupplier = null;
 
@@ -232,6 +233,11 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient {
             this.ztsUrl = ztsUrl;
         }
 
+        public Builder withErrorHandler(ClientBase.ErrorHandler errorHandler) {
+            this.errorHandler = errorHandler;
+            return this;
+        }
+
         public Builder withHostnameVerifier(HostnameVerifier hostnameVerifier) {
             this.hostnameVerifier = hostnameVerifier;
             return this;
@@ -251,7 +257,7 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient {
             if (Objects.isNull(sslContextSupplier)) {
                 throw new IllegalArgumentException("No ssl context or identity provider available to set up zts client");
             }
-            return new DefaultZtsClient(ztsUrl, sslContextSupplier, hostnameVerifier);
+            return new DefaultZtsClient(ztsUrl, sslContextSupplier, hostnameVerifier, errorHandler);
         }
     }
 }