diff options
12 files changed, 424 insertions, 85 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 5dd80961062..93243f8b8ed 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.component; import com.google.common.base.Strings; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; @@ -25,11 +26,9 @@ import static java.util.stream.Collectors.toMap; */ public class ConfigServerInfo { private final List<String> configServerHostNames; + private final URI loadBalancerEndpoint; private final Map<String, URI> configServerURIs; - private final Optional<KeyStoreOptions> keyStoreOptions; - private final Optional<KeyStoreOptions> trustStoreOptions; - private final Optional<AthenzIdentity> athenzIdentity; - private final Optional<ConfigServerConfig.Sia> siaConfig; + private final AthenzService configServerIdentity; public ConfigServerInfo(ConfigServerConfig config) { this.configServerHostNames = config.hosts(); @@ -37,18 +36,12 @@ public class ConfigServerInfo { config.scheme(), config.hosts(), config.port()); - this.keyStoreOptions = createKeyStoreOptions( - config.keyStoreConfig().path(), - config.keyStoreConfig().password().toCharArray(), - config.keyStoreConfig().type().name()); - this.trustStoreOptions = createKeyStoreOptions( - config.trustStoreConfig().path(), - config.trustStoreConfig().password().toCharArray(), - config.trustStoreConfig().type().name()); - this.athenzIdentity = createAthenzIdentity( - config.athenzDomain(), - config.serviceName()); - this.siaConfig = verifySiaConfig(config.sia()); + this.loadBalancerEndpoint = createLoadBalancerEndpoint(config.loadBalancerHost(), config.scheme(), config.port()); + this.configServerIdentity = (AthenzService) AthenzIdentities.from(config.configserverAthenzIdentity()); + } + + private static URI createLoadBalancerEndpoint(String loadBalancerHost, String scheme, int port) { + return URI.create(scheme + "://" + loadBalancerHost + ":" + port); } public List<String> getConfigServerHostNames() { @@ -68,20 +61,12 @@ public class ConfigServerInfo { return uri; } - public Optional<KeyStoreOptions> getKeyStoreOptions() { - return keyStoreOptions; - } - - public Optional<KeyStoreOptions> getTrustStoreOptions() { - return trustStoreOptions; - } - - public Optional<AthenzIdentity> getAthenzIdentity() { - return athenzIdentity; + public URI getLoadBalancerEndpoint() { + return loadBalancerEndpoint; } - public Optional<ConfigServerConfig.Sia> getSiaConfig() { - return siaConfig; + public AthenzService getConfigServerIdentity() { + return configServerIdentity; } private static Map<String, URI> createConfigServerUris( @@ -93,26 +78,4 @@ public class ConfigServerInfo { hostname -> URI.create(scheme + "://" + hostname + ":" + port))); } - private static Optional<ConfigServerConfig.Sia> verifySiaConfig(ConfigServerConfig.Sia sia) { - List<String> configParams = Arrays.asList( - sia.credentialsPath(), sia.configserverIdentityName(), sia.hostIdentityName(), sia.trustStoreFile()); - if (configParams.stream().allMatch(String::isEmpty)) { - return Optional.empty(); - } else if (configParams.stream().noneMatch(String::isEmpty)) { - return Optional.of(sia); - } else { - throw new IllegalArgumentException("Inconsistent sia config: " + sia); - } - } - - private static Optional<KeyStoreOptions> createKeyStoreOptions(String pathToKeyStore, char[] password, String type) { - return Optional.ofNullable(pathToKeyStore) - .filter(path -> !Strings.isNullOrEmpty(path)) - .map(path -> new KeyStoreOptions(Paths.get(path), password, type)); - } - - private static Optional<AthenzIdentity> createAthenzIdentity(String athenzDomain, String serviceName) { - if (Strings.isNullOrEmpty(athenzDomain) || Strings.isNullOrEmpty(serviceName)) return Optional.empty(); - return Optional.of(new AthenzService(athenzDomain, serviceName)); - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java index cf3e124277b..164ab6b3259 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; @@ -125,7 +126,8 @@ public class DockerAdminComponent implements AdminComponent { aclMaintainer, environment.get(), clock, - NODE_AGENT_SCAN_INTERVAL); + NODE_AGENT_SCAN_INTERVAL, + new AthenzCredentialsMaintainer(hostName, environment.get(), identityProvider)); NodeAdmin nodeAdmin = new NodeAdminImpl( dockerOperations, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java index fbd2b6f57a1..42729d06891 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java @@ -3,14 +3,18 @@ package com.yahoo.vespa.hosted.node.admin.component; import com.google.common.base.Strings; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; import java.net.InetAddress; +import java.net.URI; import java.net.UnknownHostException; import java.nio.file.Path; +import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Instant; @@ -38,6 +42,11 @@ public class Environment { private static final String CLOUD = "CLOUD"; private static final String LOGSTASH_NODES = "LOGSTASH_NODES"; private static final String COREDUMP_FEED_ENDPOINT = "COREDUMP_FEED_ENDPOINT"; + private static final String CERTIFICATE_DNS_SUFFIX = "CERTIFICATE_DNS_SUFFIX"; + private static final String ZTS_URI = "ZTS_URL"; + private static final String NODE_ATHENZ_IDENTITY = "NODE_ATHENZ_IDENTITY"; + private static final String ENABLE_NODE_AGENT_CERT = "ENABLE_NODE_AGENT_CERT"; + private static final String TRUST_STORE_PATH = "TRUST_STORE_PATH"; private final ConfigServerInfo configServerInfo; private final String environment; @@ -51,6 +60,11 @@ public class Environment { private final NodeType nodeType; private final String cloud; private final ContainerEnvironmentResolver containerEnvironmentResolver; + private final String certificateDnsSuffix; + private final URI ztsUri; + private final AthenzService nodeAthenzIdentity; + private final boolean nodeAgentCertEnabled; + private final Path trustStorePath; static { filenameFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -58,6 +72,7 @@ public class Environment { public Environment(ConfigServerConfig configServerConfig) { this(configServerConfig, + Paths.get(getEnvironmentVariable(TRUST_STORE_PATH)), getEnvironmentVariable(ENVIRONMENT), getEnvironmentVariable(REGION), getEnvironmentVariable(SYSTEM), @@ -68,10 +83,15 @@ public class Environment { Optional.of(getEnvironmentVariable(COREDUMP_FEED_ENDPOINT)), NodeType.host, getEnvironmentVariable(CLOUD), - new DefaultContainerEnvironmentResolver()); + new DefaultContainerEnvironmentResolver(), + getEnvironmentVariable(CERTIFICATE_DNS_SUFFIX), + URI.create(getEnvironmentVariable(ZTS_URI)), + (AthenzService)AthenzIdentities.from(getEnvironmentVariable(NODE_ATHENZ_IDENTITY)), + Boolean.valueOf(getEnvironmentVariable(ENABLE_NODE_AGENT_CERT))); } private Environment(ConfigServerConfig configServerConfig, + Path trustStorePath, String environment, String region, String system, @@ -82,7 +102,11 @@ public class Environment { Optional<String> coreDumpFeedEndpoint, NodeType nodeType, String cloud, - ContainerEnvironmentResolver containerEnvironmentResolver) { + ContainerEnvironmentResolver containerEnvironmentResolver, + String certificateDnsSuffix, + URI ztsUri, + AthenzService nodeAthenzIdentity, + boolean nodeAgentCertEnabled) { Objects.requireNonNull(configServerConfig, "configServerConfig cannot be null"); Objects.requireNonNull(environment, "environment cannot be null"); Objects.requireNonNull(region, "region cannot be null"); @@ -101,6 +125,11 @@ public class Environment { this.nodeType = nodeType; this.cloud = cloud; this.containerEnvironmentResolver = containerEnvironmentResolver; + this.certificateDnsSuffix = certificateDnsSuffix; + this.ztsUri = ztsUri; + this.nodeAthenzIdentity = nodeAthenzIdentity; + this.nodeAgentCertEnabled = nodeAgentCertEnabled; + this.trustStorePath = trustStorePath; } public List<String> getConfigServerHostNames() { return configServerInfo.getConfigServerHostNames(); } @@ -216,8 +245,32 @@ public class Environment { return containerEnvironmentResolver; } - public ConfigServerInfo getConfigServerInfo() { - return configServerInfo; + public Path getTrustStorePath() { + return trustStorePath; + } + + public AthenzService getConfigserverAthenzIdentity() { + return configServerInfo.getConfigServerIdentity(); + } + + public AthenzService getNodeAthenzIdentity() { + return nodeAthenzIdentity; + } + + public String getCertificateDnsSuffix() { + return certificateDnsSuffix; + } + + public URI getZtsUri() { + return ztsUri; + } + + public URI getConfigserverLoadBalancerEndpoint() { + return configServerInfo.getLoadBalancerEndpoint(); + } + + public boolean isNodeAgentCertEnabled() { + return nodeAgentCertEnabled; } public static class Builder { @@ -233,6 +286,11 @@ public class Environment { private NodeType nodeType = NodeType.tenant; private String cloud; private ContainerEnvironmentResolver containerEnvironmentResolver; + private String certificateDnsSuffix; + private URI ztsUri; + private AthenzService nodeAthenzIdentity; + private boolean nodeAgentCertEnabled; + private Path trustStorePath; public Builder configServerConfig(ConfigServerConfig configServerConfig) { this.configServerConfig = configServerConfig; @@ -294,8 +352,34 @@ public class Environment { return this; } + public Builder certificateDnsSuffix(String certificateDnsSuffix) { + this.certificateDnsSuffix = certificateDnsSuffix; + return this; + } + + public Builder ztsUri(URI ztsUri) { + this.ztsUri = ztsUri; + return this; + } + + public Builder nodeAthenzIdentity(AthenzService nodeAthenzIdentity) { + this.nodeAthenzIdentity = nodeAthenzIdentity; + return this; + } + + public Builder enableNodeAgentCert(boolean nodeAgentCertEnabled) { + this.nodeAgentCertEnabled = nodeAgentCertEnabled; + return this; + } + + public Builder trustStorePath(Path trustStorePath) { + this.trustStorePath = trustStorePath; + return this; + } + public Environment build() { return new Environment(configServerConfig, + trustStorePath, environment, region, system, @@ -306,7 +390,11 @@ public class Environment { coredumpFeedEndpoint, nodeType, cloud, - Optional.ofNullable(containerEnvironmentResolver).orElseGet(DefaultContainerEnvironmentResolver::new)); + Optional.ofNullable(containerEnvironmentResolver).orElseGet(DefaultContainerEnvironmentResolver::new), + certificateDnsSuffix, + ztsUri, + nodeAthenzIdentity, + nodeAgentCertEnabled); } } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 25ec4fbd1dd..12ba777f018 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -61,7 +61,7 @@ public class ConfigServerApiImpl implements ConfigServerApi { public static ConfigServerApiImpl create(ConfigServerInfo info, SiaIdentityProvider provider) { return new ConfigServerApiImpl( info.getConfigServerUris(), - new AthenzIdentityVerifier(singleton(info.getAthenzIdentity().get())), + new AthenzIdentityVerifier(singleton(info.getConfigServerIdentity())), provider); } @@ -70,7 +70,7 @@ public class ConfigServerApiImpl implements ConfigServerApi { HostName configServerHostname) { return new ConfigServerApiImpl( Collections.singleton(info.getConfigServerUri(configServerHostname.value())), - new AthenzIdentityVerifier(singleton(info.getAthenzIdentity().get())), + new AthenzIdentityVerifier(singleton(info.getConfigServerIdentity())), provider); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 64c7fedb622..ef5f2e60220 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -341,6 +341,9 @@ public class DockerOperationsImpl implements DockerOperations { directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/db/vespa"), false); directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/jdisc_container"), false); directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/jdisc_core"), false); + if (environment.getNodeType() == NodeType.host) { + directoriesToMount.put(Paths.get("/var/lib/sia"), true); + } directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/maven"), false); directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/run"), false); directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/scoreboards"), true); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java new file mode 100644 index 00000000000..869e59d890b --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -0,0 +1,283 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.identity; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; +import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; +import com.yahoo.vespa.athenz.client.zts.ZtsClient; +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; +import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; +import com.yahoo.vespa.athenz.identityprovider.client.DefaultIdentityDocumentClient; +import com.yahoo.vespa.athenz.identityprovider.client.InstanceCsrGenerator; +import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; +import com.yahoo.vespa.athenz.tls.KeyAlgorithm; +import com.yahoo.vespa.athenz.tls.KeyStoreType; +import com.yahoo.vespa.athenz.tls.KeyUtils; +import com.yahoo.vespa.athenz.tls.Pkcs10Csr; +import com.yahoo.vespa.athenz.tls.SslContextBuilder; +import com.yahoo.vespa.athenz.tls.X509CertificateUtils; +import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; + +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Set; + +import static java.util.Collections.singleton; + +/** + * A maintainer that is responsible for providing and refreshing Athenz credentials for a container. + * + * @author bjorncs + */ +@SuppressWarnings("deprecation") // TODO Use new entity response types +public class AthenzCredentialsMaintainer { + + private static final Duration EXPIRY_MARGIN = Duration.ofDays(1); + private static final Duration REFRESH_PERIOD = Duration.ofDays(1); + private static final Path CONTAINER_SIA_DIRECTORY = Paths.get("/var/lib/sia"); + + private static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); + + private final boolean enabled; + private final PrefixLogger log; + private final String hostname; + private final Path trustStorePath; + private final Path privateKeyFile; + private final Path certificateFile; + private final AthenzService containerIdentity; + private final URI ztsEndpoint; + private final Clock clock; + private final ServiceIdentityProvider hostIdentityProvider; + private final IdentityDocumentClient identityDocumentClient; + private final InstanceCsrGenerator csrGenerator; + private final AthenzService configserverIdentity; + private final String zoneRegion; + private final String zoneEnvironment; + + public AthenzCredentialsMaintainer(String hostname, + Environment environment, + ServiceIdentityProvider hostIdentityProvider) { + ContainerName containerName = ContainerName.fromHostname(hostname); + Path containerSiaDirectory = environment.pathInNodeAdminFromPathInNode(containerName, CONTAINER_SIA_DIRECTORY); + this.enabled = environment.isNodeAgentCertEnabled(); + this.log = PrefixLogger.getNodeAgentLogger(AthenzCredentialsMaintainer.class, containerName); + this.hostname = hostname; + this.containerIdentity = environment.getNodeAthenzIdentity(); + this.ztsEndpoint = environment.getZtsUri(); + this.configserverIdentity = environment.getConfigserverAthenzIdentity(); + this.csrGenerator = new InstanceCsrGenerator(environment.getCertificateDnsSuffix()); + this.trustStorePath = environment.getTrustStorePath(); + this.privateKeyFile = getPrivateKeyFile(containerSiaDirectory, containerIdentity); + this.certificateFile = getCertificateFile(containerSiaDirectory, containerIdentity); + this.hostIdentityProvider = hostIdentityProvider; + this.identityDocumentClient = + new DefaultIdentityDocumentClient( + environment.getConfigserverLoadBalancerEndpoint(), + hostIdentityProvider, + new AthenzIdentityVerifier(singleton(configserverIdentity))); + this.clock = Clock.systemUTC(); + this.zoneRegion = environment.getRegion(); + this.zoneEnvironment = environment.getEnvironment(); + } + + /** + * @param nodeSpec Node specification + * @return Returns true if credentials were updated + */ + public boolean converge(NodeSpec nodeSpec) { + try { + if (!enabled) { + log.debug("Feature disabled on this host - not fetching certificate"); + return false; + } + log.debug("Checking certificate"); + Instant now = clock.instant(); + VespaUniqueInstanceId instanceId = getVespaUniqueInstanceId(nodeSpec); + Set<String> ipAddresses = nodeSpec.getIpAddresses(); + if (!Files.exists(privateKeyFile) || !Files.exists(certificateFile)) { + log.info("Certificate and/or private key file does not exist"); + Files.createDirectories(privateKeyFile.getParent()); + Files.createDirectories(certificateFile.getParent()); + registerIdentity(instanceId, ipAddresses); + return true; + } + X509Certificate certificate = readCertificateFromFile(); + Instant expiry = certificate.getNotAfter().toInstant(); + if (isCertificateExpired(expiry, now)) { + log.info(String.format("Certificate has expired (expiry=%s)", expiry.toString())); + registerIdentity(instanceId, ipAddresses); + return true; + } + Duration age = Duration.between(certificate.getNotBefore().toInstant(), now); + if (shouldRefreshCredentials(age)) { + log.info(String.format("Certificate is ready to be refreshed (age=%s)", age.toString())); + refreshIdentity(instanceId, ipAddresses); + return true; + } + log.debug("Certificate is still valid"); + return false; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public void clearCredentials() { + if (!enabled) return; + try { + if (Files.deleteIfExists(privateKeyFile)) + log.info(String.format("Deleted private key file (path=%s)", privateKeyFile)); + if (Files.deleteIfExists(certificateFile)) + log.info(String.format("Deleted certificate file (path=%s)", certificateFile)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private VespaUniqueInstanceId getVespaUniqueInstanceId(NodeSpec nodeSpec) { + NodeSpec.Membership membership = nodeSpec.getMembership().get(); + NodeSpec.Owner owner = nodeSpec.getOwner().get(); + return new VespaUniqueInstanceId( + membership.getIndex(), + membership.getClusterId(), + owner.getInstance(), + owner.getApplication(), + owner.getTenant(), + zoneRegion, + zoneEnvironment); + } + + private boolean shouldRefreshCredentials(Duration age) { + return age.compareTo(REFRESH_PERIOD) >= 0; + } + + private X509Certificate readCertificateFromFile() throws IOException { + String pemEncodedCertificate = new String(Files.readAllBytes(certificateFile)); + return X509CertificateUtils.fromPem(pemEncodedCertificate); + } + + private boolean isCertificateExpired(Instant expiry, Instant now) { + return expiry.minus(EXPIRY_MARGIN).isAfter(now); + } + + private void registerIdentity(VespaUniqueInstanceId instanceId, Set<String> ipAddresses) { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); + Pkcs10Csr csr = csrGenerator.generateCsr(containerIdentity, instanceId, ipAddresses, keyPair); + SignedIdentityDocument signedIdentityDocument = identityDocumentClient.getNodeIdentityDocument(hostname); + try (ZtsClient ztsClient = new DefaultZtsClient(ztsEndpoint, hostIdentityProvider)) { + InstanceIdentity instanceIdentity = + ztsClient.registerInstance( + configserverIdentity, + containerIdentity, + instanceId.asDottedString(), + toAttestationDataString(signedIdentityDocument), + false, + csr); + writePrivateKeyAndCertificate(keyPair.getPrivate(), instanceIdentity.certificate()); + log.info("Instance successfully registered and credentials written to file"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (Exception e) { + // TODO Change close() in ZtsClient to not throw checked exception + throw new RuntimeException(e); + } + } + + private void refreshIdentity(VespaUniqueInstanceId instanceId, Set<String> ipAddresses) { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); + Pkcs10Csr csr = csrGenerator.generateCsr(containerIdentity, instanceId, ipAddresses, keyPair); + SSLContext containerIdentitySslContext = + new SslContextBuilder() + .withKeyStore(privateKeyFile.toFile(), certificateFile.toFile()) + .withTrustStore(trustStorePath.toFile(), KeyStoreType.JKS) + .build(); + try (ZtsClient ztsClient = new DefaultZtsClient(ztsEndpoint, containerIdentity, containerIdentitySslContext)) { + InstanceIdentity instanceIdentity = + ztsClient.refreshInstance( + configserverIdentity, + containerIdentity, + instanceId.asDottedString(), + false, + csr); + writePrivateKeyAndCertificate(keyPair.getPrivate(), instanceIdentity.certificate()); + log.info("Instance successfully refreshed and credentials written to file"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (Exception e) { + // TODO Change close() in ZtsClient to not throw checked exception + throw new RuntimeException(e); + } + } + + private void writePrivateKeyAndCertificate(PrivateKey privateKey, X509Certificate certificate) throws IOException { + Path tempPrivateKeyFile = toTempPath(privateKeyFile); + Files.write(tempPrivateKeyFile, KeyUtils.toPem(privateKey).getBytes()); + Path tempCertificateFile = toTempPath(certificateFile); + Files.write(tempCertificateFile, X509CertificateUtils.toPem(certificate).getBytes()); + + Files.move(tempPrivateKeyFile, privateKeyFile, StandardCopyOption.ATOMIC_MOVE); + Files.move(tempCertificateFile, certificateFile, StandardCopyOption.ATOMIC_MOVE); + } + + private static Path toTempPath(Path file) { + return Paths.get(file.toAbsolutePath().toString() + ".tmp"); + } + + // TODO Move to vespa-athenz + private String toAttestationDataString(SignedIdentityDocument signedIdDoc) throws JsonProcessingException { + com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument idDoc = signedIdDoc.identityDocument(); + com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocument identityDocumentPayload = + new com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocument( + com.yahoo.vespa.athenz.identityprovider.api.bindings.ProviderUniqueId.fromVespaUniqueInstanceId(idDoc.providerUniqueId()), + idDoc.configServerHostname(), + idDoc.instanceHostname(), + idDoc.createdAt(), + idDoc.ipAddresses()); + String rawIdentityDocument = objectMapper.writeValueAsString(identityDocumentPayload); + com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocument payload = + new com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocument( + rawIdentityDocument, + signedIdDoc.signature(), + signedIdDoc.signingKeyVersion(), + signedIdDoc.providerUniqueId().asDottedString(), + signedIdDoc.dnsSuffix(), + signedIdDoc.providerService().getFullName(), + signedIdDoc.ztsEndpoint(), + signedIdDoc.documentVersion()); + return objectMapper.writeValueAsString(payload); + } + + // TODO Move to vespa-athenz + private static Path getPrivateKeyFile(Path root, AthenzService service) { + return root + .resolve("keys") + .resolve(String.format("%s.%s.key.pem", service.getDomain().getName(), service.getName())); + } + + // TODO Move to vespa-athenz + private static Path getCertificateFile(Path root, AthenzService service) { + return root + .resolve("certs") + .resolve(String.format("%s.%s.cert.pem", service.getDomain().getName(), service.getName())); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/package-info.java new file mode 100644 index 00000000000..7ee04a33b05 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.maintenance.identity; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 4b37678e376..7fa9a90b744 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeReposit import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import com.yahoo.vespa.hosted.provision.Node; @@ -77,6 +78,7 @@ public class NodeAgentImpl implements NodeAgent { private final Environment environment; private final Clock clock; private final Duration timeBetweenEachConverge; + private final AthenzCredentialsMaintainer athenzCredentialsMaintainer; private final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private final LinkedList<String> debugMessages = new LinkedList<>(); @@ -121,7 +123,8 @@ public class NodeAgentImpl implements NodeAgent { final Runnable aclMaintainer, final Environment environment, final Clock clock, - final Duration timeBetweenEachConverge) { + final Duration timeBetweenEachConverge, + final AthenzCredentialsMaintainer athenzCredentialsMaintainer) { this.containerName = ContainerName.fromHostname(hostName); this.logger = PrefixLogger.getNodeAgentLogger(NodeAgentImpl.class, containerName); this.hostname = hostName; @@ -134,6 +137,7 @@ public class NodeAgentImpl implements NodeAgent { this.clock = clock; this.timeBetweenEachConverge = timeBetweenEachConverge; this.lastConverge = clock.instant(); + this.athenzCredentialsMaintainer = athenzCredentialsMaintainer; this.loopThread = new Thread(() -> { try { @@ -494,6 +498,8 @@ public class NodeAgentImpl implements NodeAgent { runLocalResumeScriptIfNeeded(node); + athenzCredentialsMaintainer.converge(node); + doBeforeConverge(node); // Because it's more important to stop a bad release from rolling out in prod, @@ -521,6 +527,7 @@ public class NodeAgentImpl implements NodeAgent { removeContainerIfNeededUpdateContainerState(node, container); logger.info("State is " + node.getState() + ", will delete application storage and mark node as ready"); storageMaintainer.cleanupNodeStorage(containerName, node); + athenzCredentialsMaintainer.clearCredentials(); updateNodeRepoWithCurrentAttributes(node); nodeRepository.setNodeState(hostname, Node.State.ready); expectNodeNotInNodeRepo = true; diff --git a/node-admin/src/main/resources/configdefinitions/config-server.def b/node-admin/src/main/resources/configdefinitions/config-server.def index 5e4d2b76a34..6a088829bad 100644 --- a/node-admin/src/main/resources/configdefinitions/config-server.def +++ b/node-admin/src/main/resources/configdefinitions/config-server.def @@ -4,26 +4,5 @@ namespace=vespa.hosted.node.admin.config hosts[] string port int default=8080 range=[1,65535] scheme string default="http" - -# TODO Remove once self-signed certs are gone -# Optional options used to authenticate config server -athenzDomain string default="" -serviceName string default="" - -# Configuration of Athenz SIA (Service Identity Agent) -sia.hostIdentityName string default="" -sia.configserverIdentityName string default="" -sia.credentialsPath string default="" -sia.trustStoreFile string default="" - -# TODO Remove once self-signed certs are gone -# Optional options about key store to use when communicating with config server -keyStoreConfig.path string default="" # Path to keystore -keyStoreConfig.type enum { JKS, PEM, PKCS12 } default=JKS -keyStoreConfig.password string default="" - -# TODO Remove once self-signed certs are gone -# Optional options about trust store to use to authenticate config server -trustStoreConfig.path string default="" -trustStoreConfig.type enum { JKS, PEM, PKCS12 } default=JKS -trustStoreConfig.password string default="" +loadBalancerHost string default="" +configserverAthenzIdentity string default="vespa.configserver"
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index fd92b24a380..1e80388d071 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; @@ -72,11 +73,12 @@ public class DockerTester implements AutoCloseable { DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment, null, new IPAddressesImpl()); StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, null, environment, callOrderVerifier, clock); AclMaintainer aclMaintainer = mock(AclMaintainer.class); + AthenzCredentialsMaintainer athenzCredentialsMaintainer = mock(AthenzCredentialsMaintainer.class); MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, - orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); + orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL, athenzCredentialsMaintainer); nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, storageMaintainer, nodeAdmin, "basehostname", clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java index 978db51aa77..319207f9e95 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; @@ -240,9 +241,10 @@ public class RunInContainerTest { private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); private final Environment environment = new Environment.Builder().build(); private final MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); + private final AthenzCredentialsMaintainer athenzCredentialsMaintainer = mock(AthenzCredentialsMaintainer.class); private final Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, dockerOperationsMock, - storageMaintainer, aclMaintainer, environment, Clock.systemUTC(), NODE_AGENT_SCAN_INTERVAL); + storageMaintainer, aclMaintainer, environment, Clock.systemUTC(), NODE_AGENT_SCAN_INTERVAL, athenzCredentialsMaintainer); private final NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperationsMock, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); private final NodeAdminStateUpdaterImpl nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, storageMaintainer, nodeAdmin, "localhost.test.yahoo.com", diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index d6d7de7f835..543a2da5448 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import com.yahoo.vespa.hosted.provision.Node; @@ -79,6 +80,7 @@ public class NodeAgentImplTest { private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); private final Docker.ContainerStats emptyContainerStats = new ContainerStatsImpl(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); + private final AthenzCredentialsMaintainer athenzCredentialsMaintainer = mock(AthenzCredentialsMaintainer.class); private final PathResolver pathResolver = mock(PathResolver.class); private final ManualClock clock = new ManualClock(); @@ -719,7 +721,7 @@ public class NodeAgentImplTest { doNothing().when(storageMaintainer).writeMetricsConfig(any(), any()); return new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, - storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); + storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL, athenzCredentialsMaintainer); } private void mockGetContainer(DockerImage dockerImage, boolean isRunning) { |