Add AthenzCredentialsMaintainer config params to Environment

4 files changed, 137 insertions, 31 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
index 5dd80961062..5f0cb595fb5 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
@@ -25,6 +25,7 @@ import static java.util.stream.Collectors.toMap;
  */
 public class ConfigServerInfo {
     private final List<String> configServerHostNames;
+    private final URI loadBalancerEndpoint;
     private final Map<String, URI> configServerURIs;
     private final Optional<KeyStoreOptions> keyStoreOptions;
     private final Optional<KeyStoreOptions> trustStoreOptions;
@@ -37,6 +38,7 @@ public class ConfigServerInfo {
                 config.scheme(),
                 config.hosts(),
                 config.port());
+        this.loadBalancerEndpoint = createLoadBalancerEndpoint(config.loadBalancerHost(), config.scheme(), config.port());
         this.keyStoreOptions = createKeyStoreOptions(
                 config.keyStoreConfig().path(),
                 config.keyStoreConfig().password().toCharArray(),
@@ -51,6 +53,10 @@ public class ConfigServerInfo {
         this.siaConfig = verifySiaConfig(config.sia());
     }
 
+    private static URI createLoadBalancerEndpoint(String loadBalancerHost, String scheme, int port) {
+        return URI.create(scheme + "://" + loadBalancerHost + ":" + port);
+    }
+
     public List<String> getConfigServerHostNames() {
         return configServerHostNames;
     }
@@ -68,6 +74,10 @@ public class ConfigServerInfo {
         return uri;
     }
 
+    public URI getLoadBalancerEndpoint() {
+        return loadBalancerEndpoint;
+    }
+
     public Optional<KeyStoreOptions> getKeyStoreOptions() {
         return keyStoreOptions;
     }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
index fbd2b6f57a1..5498e86ce4f 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
@@ -3,12 +3,15 @@ package com.yahoo.vespa.hosted.node.admin.component;
 
 import com.google.common.base.Strings;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.athenz.api.AthenzService;
+import com.yahoo.vespa.athenz.utils.AthenzIdentities;
 import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig;
 import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver;
 
 import java.net.InetAddress;
+import java.net.URI;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
 import java.text.DateFormat;
@@ -38,6 +41,10 @@ public class Environment {
     private static final String CLOUD = "CLOUD";
     private static final String LOGSTASH_NODES = "LOGSTASH_NODES";
     private static final String COREDUMP_FEED_ENDPOINT = "COREDUMP_FEED_ENDPOINT";
+    private static final String CERTIFICATE_DNS_SUFFIX = "CERTIFICATE_DNS_SUFFIX";
+    private static final String ZTS_URI = "ZTS_URL";
+    private static final String NODE_ATHENZ_IDENTITY = "NODE_ATHENZ_IDENTITY";
+    private static final String ENABLE_NODE_AGENT_CERT = "ENABLE_NODE_AGENT_CERT";
 
     private final ConfigServerInfo configServerInfo;
     private final String environment;
@@ -51,6 +58,10 @@ public class Environment {
     private final NodeType nodeType;
     private final String cloud;
     private final ContainerEnvironmentResolver containerEnvironmentResolver;
+    private final String certificateDnsSuffix;
+    private final URI ztsUri;
+    private final AthenzService nodeAthenzIdentity;
+    private final boolean nodeAgentCertEnabled;
 
     static {
         filenameFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -68,7 +79,11 @@ public class Environment {
              Optional.of(getEnvironmentVariable(COREDUMP_FEED_ENDPOINT)),
              NodeType.host,
              getEnvironmentVariable(CLOUD),
-             new DefaultContainerEnvironmentResolver());
+             new DefaultContainerEnvironmentResolver(),
+             getEnvironmentVariable(CERTIFICATE_DNS_SUFFIX),
+             URI.create(getEnvironmentVariable(ZTS_URI)),
+             (AthenzService)AthenzIdentities.from(getEnvironmentVariable(NODE_ATHENZ_IDENTITY)),
+             Boolean.valueOf(getEnvironmentVariable(ENABLE_NODE_AGENT_CERT)));
     }
 
     private Environment(ConfigServerConfig configServerConfig,
@@ -82,7 +97,11 @@ public class Environment {
                         Optional<String> coreDumpFeedEndpoint,
                         NodeType nodeType,
                         String cloud,
-                        ContainerEnvironmentResolver containerEnvironmentResolver) {
+                        ContainerEnvironmentResolver containerEnvironmentResolver,
+                        String certificateDnsSuffix,
+                        URI ztsUri,
+                        AthenzService nodeAthenzIdentity,
+                        boolean nodeAgentCertEnabled) {
         Objects.requireNonNull(configServerConfig, "configServerConfig cannot be null");
         Objects.requireNonNull(environment, "environment cannot be null");
         Objects.requireNonNull(region, "region cannot be null");
@@ -101,6 +120,10 @@ public class Environment {
         this.nodeType = nodeType;
         this.cloud = cloud;
         this.containerEnvironmentResolver = containerEnvironmentResolver;
+        this.certificateDnsSuffix = certificateDnsSuffix;
+        this.ztsUri = ztsUri;
+        this.nodeAthenzIdentity = nodeAthenzIdentity;
+        this.nodeAgentCertEnabled = nodeAgentCertEnabled;
     }
 
     public List<String> getConfigServerHostNames() { return configServerInfo.getConfigServerHostNames(); }
@@ -220,6 +243,34 @@ public class Environment {
         return configServerInfo;
     }
 
+    public Path getTrustStorePath() {
+        return configServerInfo.getTrustStoreOptions().map(o -> o.path).orElseThrow(IllegalStateException::new);
+    }
+
+    public AthenzService getConfigserverAthenzIdentity() {
+        return (AthenzService) configServerInfo.getAthenzIdentity().orElseThrow(IllegalStateException::new);
+    }
+
+    public AthenzService getNodeAthenzIdentity() {
+        return nodeAthenzIdentity;
+    }
+
+    public String getCertificateDnsSuffix() {
+        return certificateDnsSuffix;
+    }
+
+    public URI getZtsUri() {
+        return ztsUri;
+    }
+
+    public URI getConfigserverLoadBalancerEndpoint() {
+        return configServerInfo.getLoadBalancerEndpoint();
+    }
+
+    public boolean isNodeAgentCertEnabled() {
+        return nodeAgentCertEnabled;
+    }
+
     public static class Builder {
         private ConfigServerConfig configServerConfig;
         private String environment;
@@ -233,6 +284,10 @@ public class Environment {
         private NodeType nodeType = NodeType.tenant;
         private String cloud;
         private ContainerEnvironmentResolver containerEnvironmentResolver;
+        private String certificateDnsSuffix;
+        private URI ztsUri;
+        private AthenzService nodeAthenzIdentity;
+        private boolean nodeAgentCertEnabled;
 
         public Builder configServerConfig(ConfigServerConfig configServerConfig) {
             this.configServerConfig = configServerConfig;
@@ -294,6 +349,26 @@ public class Environment {
             return this;
         }
 
+        public Builder certificateDnsSuffix(String certificateDnsSuffix) {
+            this.certificateDnsSuffix = certificateDnsSuffix;
+            return this;
+        }
+
+        public Builder ztsUri(URI ztsUri) {
+            this.ztsUri = ztsUri;
+            return this;
+        }
+
+        public Builder nodeAthenzIdentity(AthenzService nodeAthenzIdentity) {
+            this.nodeAthenzIdentity = nodeAthenzIdentity;
+            return this;
+        }
+
+        public Builder enableNodeAgentCert(boolean nodeAgentCertEnabled) {
+            this.nodeAgentCertEnabled = nodeAgentCertEnabled;
+            return this;
+        }
+
         public Environment build() {
             return new Environment(configServerConfig,
                                    environment,
@@ -306,7 +381,11 @@ public class Environment {
                                    coredumpFeedEndpoint,
                                    nodeType,
                                    cloud,
-                                   Optional.ofNullable(containerEnvironmentResolver).orElseGet(DefaultContainerEnvironmentResolver::new));
+                                   Optional.ofNullable(containerEnvironmentResolver).orElseGet(DefaultContainerEnvironmentResolver::new),
+                                   certificateDnsSuffix,
+                                   ztsUri,
+                                   nodeAthenzIdentity,
+                                   nodeAgentCertEnabled);
         }
     }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
index dcc34d0306e..84f41a3e25b 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
@@ -12,8 +12,9 @@ import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
 import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient;
 import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument;
 import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId;
-import com.yahoo.vespa.athenz.identityprovider.api.bindings.ProviderUniqueId;
+import com.yahoo.vespa.athenz.identityprovider.client.DefaultIdentityDocumentClient;
 import com.yahoo.vespa.athenz.identityprovider.client.InstanceCsrGenerator;
+import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
 import com.yahoo.vespa.athenz.tls.KeyAlgorithm;
 import com.yahoo.vespa.athenz.tls.KeyStoreType;
 import com.yahoo.vespa.athenz.tls.KeyUtils;
@@ -41,6 +42,8 @@ import java.time.Duration;
 import java.time.Instant;
 import java.util.Set;
 
+import static java.util.Collections.singleton;
+
 /**
  * A maintainer that is responsible for providing and refreshing Athenz credentials for a container.
  *
@@ -51,12 +54,13 @@ public class AthenzCredentialsMaintainer {
 
     private static final Duration EXPIRY_MARGIN = Duration.ofDays(1);
     private static final Duration REFRESH_PERIOD = Duration.ofDays(1);
+    private static final Path CONTAINER_SIA_DIRECTORY = Paths.get("/var/lib/sia");
 
     private static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());
 
+    private final boolean enabled;
     private final PrefixLogger log;
     private final String hostname;
-    private final Environment environment;
     private final Path trustStorePath;
     private final Path privateKeyFile;
     private final Path certificateFile;
@@ -67,30 +71,33 @@ public class AthenzCredentialsMaintainer {
     private final IdentityDocumentClient identityDocumentClient;
     private final InstanceCsrGenerator csrGenerator;
     private final AthenzService configserverIdentity;
+    private final String zoneRegion;
+    private final String zoneEnvironment;
 
-    AthenzCredentialsMaintainer(String hostname,
-                                Path trustStorePath,
-                                Environment environment,
-                                Path containerSiaDirectory,
-                                URI ztsEndpoint,
-                                String dnsSuffix,
-                                AthenzService configserverIdentity,
-                                AthenzService containerIdentity,
-                                ServiceIdentityProvider hostIdentityProvider,
-                                IdentityDocumentClient identityDocumentClient) {
-        this.log = PrefixLogger.getNodeAgentLogger(AthenzCredentialsMaintainer.class, ContainerName.fromHostname(hostname));
+    public AthenzCredentialsMaintainer(String hostname,
+                                       Environment environment,
+                                       ServiceIdentityProvider hostIdentityProvider) {
+        ContainerName containerName = ContainerName.fromHostname(hostname);
+        Path containerSiaDirectory = environment.pathInNodeAdminFromPathInNode(containerName, CONTAINER_SIA_DIRECTORY);
+        this.enabled = environment.isNodeAgentCertEnabled();
+        this.log = PrefixLogger.getNodeAgentLogger(AthenzCredentialsMaintainer.class, containerName);
         this.hostname = hostname;
-        this.environment = environment;
-        this.containerIdentity = containerIdentity;
-        this.ztsEndpoint = ztsEndpoint;
-        this.configserverIdentity = configserverIdentity;
-        this.csrGenerator = new InstanceCsrGenerator(dnsSuffix);
-        this.trustStorePath = trustStorePath;
+        this.containerIdentity = environment.getNodeAthenzIdentity();
+        this.ztsEndpoint = environment.getZtsUri();
+        this.configserverIdentity = environment.getConfigserverAthenzIdentity();
+        this.csrGenerator = new InstanceCsrGenerator(environment.getCertificateDnsSuffix());
+        this.trustStorePath = environment.getTrustStorePath();
         this.privateKeyFile = getPrivateKeyFile(containerSiaDirectory, containerIdentity);
         this.certificateFile = getCertificateFile(containerSiaDirectory, containerIdentity);
         this.hostIdentityProvider = hostIdentityProvider;
-        this.identityDocumentClient = identityDocumentClient;
+        this.identityDocumentClient =
+                new DefaultIdentityDocumentClient(
+                        environment.getConfigserverLoadBalancerEndpoint(),
+                        hostIdentityProvider,
+                        new AthenzIdentityVerifier(singleton(configserverIdentity)));
         this.clock = Clock.systemUTC();
+        this.zoneRegion = environment.getRegion();
+        this.zoneEnvironment = environment.getEnvironment();
     }
 
     /**
@@ -99,11 +106,15 @@ public class AthenzCredentialsMaintainer {
      */
     public boolean converge(NodeSpec nodeSpec) {
         try {
+            if (!enabled) {
+                log.debug("Feature disabled on this host - not fetching certificate");
+                return false;
+            }
             log.debug("Checking certificate");
             Instant now = clock.instant();
             VespaUniqueInstanceId instanceId = getVespaUniqueInstanceId(nodeSpec);
             Set<String> ipAddresses = nodeSpec.getIpAddresses();
-            if (!privateKeyFile.toFile().exists() || !certificateFile.toFile().exists()) {
+            if (!Files.exists(privateKeyFile) || !Files.exists(certificateFile)) {
                 log.info("Certificate and/or private key file does not exist");
                 Files.createDirectories(privateKeyFile.getParent());
                 Files.createDirectories(certificateFile.getParent());
@@ -131,10 +142,15 @@ public class AthenzCredentialsMaintainer {
     }
 
     public void clearCredentials() {
-        boolean privateKeyDeleteResult = privateKeyFile.toFile().delete();
-        log.info(String.format("Deleted private key file (path=%s, result=%s)", privateKeyFile, privateKeyDeleteResult));
-        boolean certificateDeleteResult = certificateFile.toFile().delete();
-        log.info(String.format("Deleted certificate file (path=%s, result=%s)", certificateFile, certificateDeleteResult));
+        if (!enabled) return;
+        try {
+            if (Files.deleteIfExists(privateKeyFile))
+                log.info(String.format("Deleted private key file (path=%s)", privateKeyFile));
+            if (Files.deleteIfExists(certificateFile))
+                log.info(String.format("Deleted certificate file (path=%s)", certificateFile));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
     }
 
     private VespaUniqueInstanceId getVespaUniqueInstanceId(NodeSpec nodeSpec) {
@@ -146,8 +162,8 @@ public class AthenzCredentialsMaintainer {
                 owner.getInstance(),
                 owner.getApplication(),
                 owner.getTenant(),
-                environment.getRegion(),
-                environment.getEnvironment());
+                zoneRegion,
+                zoneEnvironment);
     }
 
     private boolean shouldRefreshCredentials(Duration age) {
@@ -229,7 +245,7 @@ public class AthenzCredentialsMaintainer {
         com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument idDoc = signedIdDoc.identityDocument();
         com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocument identityDocumentPayload =
                 new com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocument(
-                        ProviderUniqueId.fromVespaUniqueInstanceId(idDoc.providerUniqueId()),
+                        com.yahoo.vespa.athenz.identityprovider.api.bindings.ProviderUniqueId.fromVespaUniqueInstanceId(idDoc.providerUniqueId()),
                         idDoc.configServerHostname(),
                         idDoc.instanceHostname(),
                         idDoc.createdAt(),
diff --git a/node-admin/src/main/resources/configdefinitions/config-server.def b/node-admin/src/main/resources/configdefinitions/config-server.def
index 5e4d2b76a34..1fcf4bb0a62 100644
--- a/node-admin/src/main/resources/configdefinitions/config-server.def
+++ b/node-admin/src/main/resources/configdefinitions/config-server.def
@@ -4,6 +4,7 @@ namespace=vespa.hosted.node.admin.config
 hosts[]                      string
 port                         int                         default=8080 range=[1,65535]
 scheme                       string                      default="http"
+loadBalancerHost             string                      default=""
 
 # TODO Remove once self-signed certs are gone
 # Optional options used to authenticate config server