diff options
27 files changed, 294 insertions, 491 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java deleted file mode 100644 index a1fae9bb148..00000000000 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.yahoo.config.model.api; - -public class EndpointCertificateMetadata { - - private final String keyName; - private final String certName; - private final int version; - - public EndpointCertificateMetadata(String keyName, String certName, int version) { - this.keyName = keyName; - this.certName = certName; - this.version = version; - } - - public String keyName() { - return keyName; - } - - public String certName() { - return certName; - } - - public int version() { - return version; - } - - @Override - public String toString() { - return "EndpointCertificateMetadata{" + - "keyName='" + keyName + '\'' + - ", certName='" + certName + '\'' + - ", version=" + version + - '}'; - } -} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index d45c7ce4137..323aa473580 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -55,7 +55,7 @@ public interface ModelContext { default boolean useDedicatedNodeForLogserver() { return true; } boolean useAdaptiveDispatch(); // TODO: Remove temporary default implementation - default Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return Optional.empty(); } + default Optional<TlsSecrets> tlsSecrets() { return Optional.empty(); } double defaultTermwiseLimit(); boolean useBucketSpaceMetric(); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java index 6fcbac4f422..6a8b5a237ab 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java @@ -1,17 +1,17 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; -public class EndpointCertificateSecrets { - public static final EndpointCertificateSecrets MISSING = new EndpointCertificateSecrets(); +public class TlsSecrets { + public static final TlsSecrets MISSING = new TlsSecrets(); private final String certificate; private final String key; - private EndpointCertificateSecrets() { + private TlsSecrets() { this(null, null); } - public EndpointCertificateSecrets(String certificate, String key) { + public TlsSecrets(String certificate, String key) { this.certificate = certificate; this.key = key; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 7c9e930bb4f..b286b94c699 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -15,7 +15,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -255,7 +255,7 @@ public class DeployState implements ConfigDefinitionStore { public Instant now() { return now; } - public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return properties.endpointCertificateSecrets(); } + public Optional<TlsSecrets> tlsSecrets() { return properties.tlsSecrets(); } public Optional<String> tlsClientAuthority() { var caFile = applicationPackage.getClientSecurityFile(); @@ -289,6 +289,7 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; + private Optional<TlsSecrets> tlsSecrets = Optional.empty(); public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 73f4b71be5e..9d561a79c75 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -5,7 +5,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; @@ -39,7 +39,7 @@ public class TestProperties implements ModelContext.Properties { private boolean useDedicatedNodeForLogserver = false; private boolean useAdaptiveDispatch = false; private double defaultTermwiseLimit = 1.0; - private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); + private Optional<TlsSecrets> tlsSecrets = Optional.empty(); @Override public boolean multitenant() { return multitenant; } @@ -56,7 +56,7 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } - @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } + @Override public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useBucketSpaceMetric() { return true; } @@ -96,8 +96,8 @@ public class TestProperties implements ModelContext.Properties { } - public TestProperties setEndpointCertificateSecrets(Optional<EndpointCertificateSecrets> endpointCertificateSecrets) { - this.endpointCertificateSecrets = endpointCertificateSecrets; + public TestProperties setTlsSecrets(Optional<TlsSecrets> tlsSecrets) { + this.tlsSecrets = tlsSecrets; return this; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java index f00ad0f0dbb..2f972b8ecb3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java @@ -1,17 +1,17 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.CertificateNotReadyException; import com.yahoo.vespa.model.VespaModel; -public class EndpointCertificateSecretsValidator extends Validator { +public class TlsSecretsValidator extends Validator { /** This check is delayed until validation to allow node provisioning to complete while we are waiting for cert */ @Override public void validate(VespaModel model, DeployState deployState) { - if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) { + if (deployState.tlsSecrets().isPresent() && deployState.tlsSecrets().get() == TlsSecrets.MISSING) { throw new CertificateNotReadyException("TLS enabled, but could not retrieve certificate yet"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 1e4a45428b8..8eabc61f71f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -57,7 +57,7 @@ public class Validation { new DeploymentSpecValidator().validate(model, deployState); new RankingConstantsValidator().validate(model, deployState); new SecretStoreValidator().validate(model, deployState); - new EndpointCertificateSecretsValidator().validate(model, deployState); + new TlsSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); List<ConfigChangeAction> result = Collections.emptyList(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index efd00528d54..5e0dde6161d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -5,6 +5,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.BundlesConfig; @@ -54,6 +55,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private ContainerModelEvaluation modelEvaluation; + private Optional<TlsSecrets> tlsSecrets; private Optional<String> tlsClientAuthority; private MbusParams mbusParams; @@ -63,6 +65,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { super(parent, subId, name, deployState); + + this.tlsSecrets = deployState.tlsSecrets(); this.tlsClientAuthority = deployState.tlsClientAuthority(); restApiGroup = new ConfigProducerGroup<>(this, "rest-api"); servletGroup = new ConfigProducerGroup<>(this, "servlet"); @@ -201,6 +205,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } + public Optional<TlsSecrets> getTlsSecrets() { + return tlsSecrets; + } + public Optional<String> getTlsClientAuthority() { return tlsClientAuthority; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 12db3b87243..7a08a3c1a7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http.ssl; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; import com.yahoo.vespa.model.container.component.SimpleComponent; @@ -23,15 +23,15 @@ public class HostedSslConnectorFactory extends ConnectorFactory { /** * Create connector factory that uses a certificate provided by the config-model / configserver. */ - public static HostedSslConnectorFactory withProvidedCertificate(String serverName, EndpointCertificateSecrets endpointCertificateSecrets) { - return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificates*/null), false); + public static HostedSslConnectorFactory withProvidedCertificate(String serverName, TlsSecrets tlsSecrets) { + return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, tlsSecrets, /*tlsCaCertificates*/null), false); } /** * Create connector factory that uses a certificate provided by the config-model / configserver and a truststore configured by the application. */ - public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { - return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, tlsCaCertificates), true); + public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(String serverName, TlsSecrets tlsSecrets, String tlsCaCertificates) { + return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, tlsSecrets, tlsCaCertificates), true); } /** @@ -47,11 +47,11 @@ public class HostedSslConnectorFactory extends ConnectorFactory { } private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider( - String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { + String serverName, TlsSecrets tlsSecrets, String tlsCaCertificates) { return new ConfiguredDirectSslProvider( serverName, - endpointCertificateSecrets.key(), - endpointCertificateSecrets.certificate(), + tlsSecrets.key(), + tlsSecrets.certificate(), /*caCertificatePath*/null, tlsCaCertificates, ClientAuth.Enum.WANT_AUTH); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index aef2697a5dd..3da0b01f614 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -13,7 +13,7 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.application.provider.IncludeDirs; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; @@ -327,15 +327,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String serverName = server.getComponentId().getName(); // If the deployment contains certificate/private key reference, setup TLS port - if (deployState.endpointCertificateSecrets().isPresent()) { + if (deployState.tlsSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { throw new RuntimeException("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/security-model#data-plane"); } - EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); + TlsSecrets tlsSecrets = deployState.tlsSecrets().get(); HostedSslConnectorFactory connectorFactory = authorizeClient - ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) - : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets); + ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, tlsSecrets, deployState.tlsClientAuthority().get()) + : HostedSslConnectorFactory.withProvidedCertificate(serverName, tlsSecrets); server.addConnector(connectorFactory); } else { server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidatorTest.java index 21df39ebde8..cdb4ce955e2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidatorTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; /** * @author andreer */ -public class EndpointCertificateSecretsValidatorTest { +public class TlsSecretsValidatorTest { @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @@ -43,21 +43,21 @@ public class EndpointCertificateSecretsValidatorTest { @Test public void missing_certificate_fails_validation() throws Exception { - DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(EndpointCertificateSecrets.MISSING)); + DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(TlsSecrets.MISSING)); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); exceptionRule.expect(CertificateNotReadyException.class); exceptionRule.expectMessage("TLS enabled, but could not retrieve certificate yet"); - new EndpointCertificateSecretsValidator().validate(model, deployState); + new TlsSecretsValidator().validate(model, deployState); } @Test public void validation_succeeds_with_certificate() throws Exception { - DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(new EndpointCertificateSecrets("cert", "key"))); + DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(new TlsSecrets("cert", "key"))); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new EndpointCertificateSecretsValidator().validate(model, deployState); + new TlsSecretsValidator().validate(model, deployState); } @Test @@ -65,10 +65,10 @@ public class EndpointCertificateSecretsValidatorTest { DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.empty()); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new EndpointCertificateSecretsValidator().validate(model, deployState); + new TlsSecretsValidator().validate(model, deployState); } - private static DeployState deployState(String servicesXml, String deploymentXml, Optional<EndpointCertificateSecrets> endpointCertificateSecretsSecrets) { + private static DeployState deployState(String servicesXml, String deploymentXml, Optional<TlsSecrets> tlsSecrets) { ApplicationPackage app = new MockApplicationPackage.Builder() .withServices(servicesXml) .withDeploymentSpec(deploymentXml) @@ -79,7 +79,7 @@ public class EndpointCertificateSecretsValidatorTest { .properties( new TestProperties() .setHostedVespa(true) - .setEndpointCertificateSecrets(endpointCertificateSecretsSecrets)); + .setTlsSecrets(tlsSecrets)); final DeployState deployState = builder.build(); assertTrue("Test must emulate a hosted deployment.", deployState.isHosted()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 1bbc4ea2684..54d1c1c9793 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -5,7 +5,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -693,7 +693,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .properties( new TestProperties() .setHostedVespa(true) - .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))) + .setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))) .zone(new Zone(SystemName.Public, Environment.prod, RegionName.defaultName())) .build(); createModel(root, state, null, clusterElem); @@ -772,13 +772,13 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void requireThatProvidingEndpointCertificateSecretsOpensPort4443() { + public void requireThatProvidingTlsSecretOpensPort4443() { Element clusterElem = DomBuilderTest.parse( "<container version='1.0'>", nodesXml, "</container>" ); - DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))).build(); + DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))).build(); createModel(root, state, null, clusterElem); ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 68f507c810d..863781073f8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -258,7 +258,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas .properties( new TestProperties() .setHostedVespa(true) - .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))) + .setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))) .modelHostProvisioner(new HostsXmlProvisioner(new StringReader(hostsxml))) .build(); MockRoot root = new MockRoot("root", deployState); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 4f847f86c0c..52d47a9398b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -11,7 +11,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; @@ -130,7 +130,7 @@ public class ModelContextImpl implements ModelContext { private final boolean isBootstrap; private final boolean isFirstTimeDeployment; private final boolean useAdaptiveDispatch; - private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; + private final Optional<TlsSecrets> tlsSecrets; private final double defaultTermwiseLimit; private final boolean useBucketSpaceMetric; @@ -146,7 +146,7 @@ public class ModelContextImpl implements ModelContext { boolean isBootstrap, boolean isFirstTimeDeployment, FlagSource flagSource, - Optional<EndpointCertificateSecrets> endpointCertificateSecrets) { + Optional<TlsSecrets> tlsSecrets) { this.applicationId = applicationId; this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant"); this.configServerSpecs = configServerSpecs; @@ -160,7 +160,7 @@ public class ModelContextImpl implements ModelContext { this.isFirstTimeDeployment = isFirstTimeDeployment; this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.endpointCertificateSecrets = endpointCertificateSecrets; + this.tlsSecrets = tlsSecrets; defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useBucketSpaceMetric = Flags.USE_BUCKET_SPACE_METRIC.bindTo(flagSource) @@ -208,7 +208,7 @@ public class ModelContextImpl implements ModelContext { public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override - public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } + public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index a2fc2bfd6a0..bc6419f230f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -27,9 +27,8 @@ import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; -import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; +import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; @@ -136,10 +135,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { false, // We may be bootstrapping, but we only know and care during prepare false, // Always false, assume no one uses it when activating flagSource, - new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant)) - .readEndpointCertificateMetadata(applicationId) - .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets)); - + new TlsSecretsKeys(curator, TenantRepository.getTenantPath(tenant), secretStore).readTlsSecretsKeyFromZookeeper(applicationId)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index 1a41c1efd7a..ab3e0e863ce 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; @@ -12,7 +11,6 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; -import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; import java.time.Clock; import java.time.Duration; @@ -34,7 +32,6 @@ public final class PrepareParams { static final String VESPA_VERSION_PARAM_NAME = "vespaVersion"; static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints"; static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName"; - static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -45,12 +42,10 @@ public final class PrepareParams { private final Optional<Version> vespaVersion; private final List<ContainerEndpoint> containerEndpoints; private final Optional<String> tlsSecretsKeyName; - private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, - List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, - Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { + List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName) { this.timeoutBudget = timeoutBudget; this.applicationId = applicationId; this.ignoreValidationErrors = ignoreValidationErrors; @@ -60,7 +55,6 @@ public final class PrepareParams { this.vespaVersion = vespaVersion; this.containerEndpoints = containerEndpoints; this.tlsSecretsKeyName = tlsSecretsKeyName; - this.endpointCertificateMetadata = endpointCertificateMetadata; } public static class Builder { @@ -74,7 +68,6 @@ public final class PrepareParams { private Optional<Version> vespaVersion = Optional.empty(); private List<ContainerEndpoint> containerEndpoints = List.of(); private Optional<String> tlsSecretsKeyName = Optional.empty(); - private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty(); public Builder() { } @@ -135,16 +128,9 @@ public final class PrepareParams { return this; } - public Builder endpointCertificateMetadata(String serialized) { - if(serialized == null) return this; - Slime slime = SlimeUtils.jsonToSlime(serialized); - endpointCertificateMetadata = Optional.of(EndpointCertificateMetadataSerializer.fromSlime(slime.get())); - return this; - } - public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, - verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata); + verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName); } } @@ -158,7 +144,6 @@ public final class PrepareParams { .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME)) .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME)) - .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) .build(); } @@ -215,8 +200,4 @@ public final class PrepareParams { public Optional<String> tlsSecretsKeyName() { return tlsSecretsKeyName; } - - public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() { - return endpointCertificateMetadata; - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 0115876ded9..171eab35507 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -12,9 +12,8 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -34,9 +33,7 @@ import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; -import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; -import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; -import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; +import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import org.xml.sax.SAXException; @@ -116,7 +113,7 @@ public class SessionPreparer { preparation.makeResult(allocatedHosts); if ( ! params.isDryRun()) { preparation.writeStateZK(); - preparation.writeEndpointCertificateMetadataZK(); + preparation.writeTlsZK(); preparation.writeContainerEndpointsZK(); preparation.distribute(); } @@ -145,10 +142,8 @@ public class SessionPreparer { final ContainerEndpointsCache containerEndpoints; final Set<ContainerEndpoint> endpointsSet; final ModelContext.Properties properties; - private final EndpointCertificateMetadataStore endpointCertificateMetadataStore; - private final EndpointCertificateRetriever endpointCertificateRetriever; - private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; - private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; + private final TlsSecretsKeys tlsSecretsKeys; + private final Optional<TlsSecrets> tlsSecrets; private ApplicationPackage applicationPackage; private List<PreparedModelsBuilder.PreparedModelResult> modelResultList; @@ -167,16 +162,8 @@ public class SessionPreparer { this.applicationId = params.getApplicationId(); this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion); this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator); - this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); - this.endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); - - this.endpointCertificateMetadata = params.endpointCertificateMetadata() - .or(() -> params.tlsSecretsKeyName().map(EndpointCertificateMetadataSerializer::fromString)); - - endpointCertificateSecrets = endpointCertificateMetadata - .or(() -> endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId)) - .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); - + this.tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore); + this.tlsSecrets = tlsSecretsKeys.getTlsSecrets(params.tlsSecretsKeyName(), applicationId); this.endpointsSet = getEndpoints(params.containerEndpoints()); this.properties = new ModelContextImpl.Properties(params.getApplicationId(), @@ -191,7 +178,7 @@ public class SessionPreparer { params.isBootstrap(), ! currentActiveApplicationSet.isPresent(), context.getFlagSource(), - endpointCertificateSecrets); + tlsSecrets); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, @@ -246,10 +233,9 @@ public class SessionPreparer { checkTimeout("write state to zookeeper"); } - void writeEndpointCertificateMetadataZK() { - endpointCertificateMetadata.ifPresent(metadata -> - endpointCertificateMetadataStore.writeEndpointCertificateMetadata(applicationId, metadata)); - checkTimeout("write endpoint certificate metadata to zookeeper"); + void writeTlsZK() { + tlsSecretsKeys.writeTlsSecretsKeyToZooKeeper(applicationId, params.tlsSecretsKeyName().orElse(null)); + checkTimeout("write tlsSecretsKey to zookeeper"); } void writeContainerEndpointsZK() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java deleted file mode 100644 index 6d092aaa18b..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.model.api.EndpointCertificateMetadata; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; - -/** - * (de)serializes endpoint certificate metadata - * - * @author andreer - */ -public class EndpointCertificateMetadataSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private final static String keyNameField = "keyName"; - private final static String certNameField = "certName"; - private final static String versionField = "version"; - - public static void toSlime(EndpointCertificateMetadata metadata, Cursor object) { - object.setString(keyNameField, metadata.keyName()); - object.setString(certNameField, metadata.certName()); - object.setLong(versionField, metadata.version()); - } - - public static EndpointCertificateMetadata fromSlime(Inspector inspector) { - switch (inspector.type()) { - case STRING: // TODO: Remove once all are transmitted and stored as JSON - return new EndpointCertificateMetadata( - inspector.asString() + "-key", - inspector.asString() + "-cert", - 0 - ); - case OBJECT: - return new EndpointCertificateMetadata( - inspector.field(keyNameField).asString(), - inspector.field(certNameField).asString(), - Math.toIntExact(inspector.field(versionField).asLong()) - ); - - default: - throw new IllegalArgumentException("Unknown format encountered for TLS secrets metadata!"); - } - } - - public static EndpointCertificateMetadata fromString(String tlsSecretsKeys) { - return fromSlime(new Slime().setString(tlsSecretsKeys)); - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java deleted file mode 100644 index 6500449e557..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.model.api.EndpointCertificateMetadata; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.path.Path; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.transaction.CuratorOperations; -import com.yahoo.vespa.curator.transaction.CuratorTransaction; - -import java.util.Optional; - -/** - * Stores the endpoint certificate metadata for an application. - * This metadata is then used to retrieve the actual secrets from {@link EndpointCertificateRetriever}. - * - * @author andreer - */ -public class EndpointCertificateMetadataStore { - - private final Path path; - private final Curator curator; - - public EndpointCertificateMetadataStore(Curator curator, Path tenantPath) { - this.curator = curator; - this.path = tenantPath.append("tlsSecretsKeys/"); - } - - /** Reads the endpoint certificate metadata from ZooKeeper, if it exists */ - public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId application) { - try { - Optional<byte[]> data = curator.getData(endpointCertificateMetadataPathOf(application)); - if (data.isEmpty() || data.get().length == 0) return Optional.empty(); - Slime slime = SlimeUtils.jsonToSlime(data.get()); - EndpointCertificateMetadata endpointCertificateMetadata = EndpointCertificateMetadataSerializer.fromSlime(slime.get()); - return Optional.of(endpointCertificateMetadata); - } catch (Exception e) { - throw new RuntimeException("Error reading TLS secret key of " + application, e); - } - } - - /** Writes the endpoint certificate metadata to ZooKeeper */ - public void writeEndpointCertificateMetadata(ApplicationId application, EndpointCertificateMetadata endpointCertificateMetadata) { - try { - Slime slime = new Slime(); - EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata, slime.setObject()); - curator.set(endpointCertificateMetadataPathOf(application), SlimeUtils.toJsonBytes(slime)); - } catch (Exception e) { - throw new RuntimeException("Could not write TLS secret key of " + application, e); - } - } - - /** Returns a transaction which deletes these tls secrets key if they exist */ - public CuratorTransaction delete(ApplicationId application) { - if (!curator.exists(endpointCertificateMetadataPathOf(application))) return CuratorTransaction.empty(curator); - return CuratorTransaction.from(CuratorOperations.delete(endpointCertificateMetadataPathOf(application).getAbsolute()), curator); - } - - /** Returns the path storing the tls secrets key for an application */ - private Path endpointCertificateMetadataPathOf(ApplicationId application) { - return path.append(application.serializedForm()); - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java deleted file mode 100644 index 5f40e5e1411..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.model.api.EndpointCertificateMetadata; -import com.yahoo.config.model.api.EndpointCertificateSecrets; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.X509CertificateUtils; - -import java.security.PrivateKey; -import java.security.PublicKey; -import java.security.cert.X509Certificate; -import java.util.Optional; - -/** - * Used to retrieve actual endpoint certificate/key from secret store. - * - * @author andreer - */ -public class EndpointCertificateRetriever { - - private final SecretStore secretStore; - - public EndpointCertificateRetriever(SecretStore secretStore) { - this.secretStore = secretStore; - } - - public Optional<EndpointCertificateSecrets> readEndpointCertificateSecrets(EndpointCertificateMetadata metadata) { - return Optional.of(readFromSecretStore(metadata)); - } - - private EndpointCertificateSecrets readFromSecretStore(EndpointCertificateMetadata endpointCertificateMetadata) { - try { - String cert = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version()); - String key = secretStore.getSecret(endpointCertificateMetadata.keyName(), endpointCertificateMetadata.version()); - - verifyKeyMatchesCertificate(endpointCertificateMetadata, cert, key); - - return new EndpointCertificateSecrets(cert, key); - } catch (RuntimeException e) { - // Assume not ready yet - return EndpointCertificateSecrets.MISSING; - } - } - - private void verifyKeyMatchesCertificate(EndpointCertificateMetadata endpointCertificateMetadata, String cert, String key) { - X509Certificate x509Certificate = X509CertificateUtils.fromPem(cert); - - PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(key); - PublicKey publicKey = x509Certificate.getPublicKey(); - - if(!X509CertificateUtils.privateKeyMatchesPublicKey(privateKey, publicKey)) { - throw new IllegalArgumentException("Failed to retrieve endpoint secrets: Certificate and key data do not match for " + endpointCertificateMetadata); - } - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java new file mode 100644 index 00000000000..da6fc490da9 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java @@ -0,0 +1,136 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.path.Path; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; + +import java.util.Optional; + +/** + * TLS Secret keys for applications (used to retrieve actual certificate/key from secret store). Persisted in ZooKeeper. + * + * @author andreer + */ +public class TlsSecretsKeys { + + private final Path path; + private final SecretStore secretStore; + private final Curator curator; + + public TlsSecretsKeys(Curator curator, Path tenantPath, SecretStore secretStore) { + this.curator = curator; + this.path = tenantPath.append("tlsSecretsKeys/"); + this.secretStore = secretStore; + } + + public Optional<TlsSecrets> readTlsSecretsKeyFromZookeeper(ApplicationId application) { + try { + Optional<byte[]> data = curator.getData(tlsSecretsKeyOf(application)); + if (data.isEmpty() || data.get().length == 0) return Optional.empty(); + + Slime slime = SlimeUtils.jsonToSlime(data.get()); + final var inspector = slime.get(); + + switch (inspector.type()) { + case STRING: // TODO: Remove once all are stored as JSON + return readFromSecretStore(Optional.ofNullable(inspector.asString())); + case OBJECT: + var tlsSecretsInfo = new TlsSecretsMetadata(); + tlsSecretsInfo.certName = inspector.field("certName").asString(); + tlsSecretsInfo.keyName = inspector.field("keyName").asString(); + tlsSecretsInfo.version = Math.toIntExact(inspector.field("version").asLong()); + return Optional.of(readFromSecretStore(tlsSecretsInfo)); + default: + throw new IllegalArgumentException("Unknown format encountered for TLS secrets metadata!"); + } + } catch (Exception e) { + throw new RuntimeException("Error reading TLS secret key of " + application, e); + } + } + + public void writeTlsSecretsKeyToZooKeeper(ApplicationId application, String tlsSecretsKey) { + if (tlsSecretsKey == null) return; + writeTlsSecretsAsString(application, tlsSecretsKey); + } + + private void writeTlsSecretsAsString(ApplicationId application, String tlsSecretsKey) { + try { + Slime slime = new Slime(); + slime.setString(tlsSecretsKey); + curator.set(tlsSecretsKeyOf(application), SlimeUtils.toJsonBytes(slime)); + } catch (Exception e) { + throw new RuntimeException("Could not write TLS secret key of " + application, e); + } + } + + void writeTlsSecretsMetadata(ApplicationId application, TlsSecretsMetadata tlsSecretsMetadata) { + try { + Slime slime = new Slime(); + Cursor cursor = slime.setObject(); + cursor.setString(TlsSecretsMetadata.certNameField, tlsSecretsMetadata.certName); + cursor.setString(TlsSecretsMetadata.keyNameField, tlsSecretsMetadata.keyName); + cursor.setLong(TlsSecretsMetadata.versionField, tlsSecretsMetadata.version); + curator.set(tlsSecretsKeyOf(application), SlimeUtils.toJsonBytes(slime)); + } catch (Exception e) { + throw new RuntimeException("Could not write TLS secret key of " + application, e); + } + } + + public Optional<TlsSecrets> getTlsSecrets(Optional<String> secretKeyname, ApplicationId applicationId) { + if (secretKeyname == null || secretKeyname.isEmpty()) { + return readTlsSecretsKeyFromZookeeper(applicationId); + } + return readFromSecretStore(secretKeyname); + } + + private Optional<TlsSecrets> readFromSecretStore(Optional<String> secretKeyname) { + if (secretKeyname.isEmpty()) return Optional.empty(); + try { + String cert = secretStore.getSecret(secretKeyname.get() + "-cert"); + String key = secretStore.getSecret(secretKeyname.get() + "-key"); + return Optional.of(new TlsSecrets(cert, key)); + } catch (RuntimeException e) { + // Assume not ready yet + return Optional.of(TlsSecrets.MISSING); + } + } + + private TlsSecrets readFromSecretStore(TlsSecretsMetadata tlsSecretsMetadata) { + try { + String cert = secretStore.getSecret(tlsSecretsMetadata.certName, tlsSecretsMetadata.version); + String key = secretStore.getSecret(tlsSecretsMetadata.keyName, tlsSecretsMetadata.version); + return new TlsSecrets(cert, key); + } catch (RuntimeException e) { + // Assume not ready yet + return TlsSecrets.MISSING; + } + } + + /** Returns a transaction which deletes these tls secrets key if they exist */ + public CuratorTransaction delete(ApplicationId application) { + if (!curator.exists(tlsSecretsKeyOf(application))) return CuratorTransaction.empty(curator); + return CuratorTransaction.from(CuratorOperations.delete(tlsSecretsKeyOf(application).getAbsolute()), curator); + } + + /** Returns the path storing the tls secrets key for an application */ + private Path tlsSecretsKeyOf(ApplicationId application) { + return path.append(application.serializedForm()); + } + + static class TlsSecretsMetadata { + final static String keyNameField = "keyName"; + final static String certNameField = "certName"; + final static String versionField = "version"; + String keyName; + String certName; + int version; + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java index 12f48778144..8a77b53875e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java @@ -7,26 +7,22 @@ import java.util.HashMap; import java.util.Map; public class MockSecretStore implements SecretStore { - Map<String, Map<Integer, String>> secrets = new HashMap<>(); + Map<String, String> secrets = new HashMap<>(); @Override public String getSecret(String key) { if(secrets.containsKey(key)) - return secrets.get(key).get(0); + return secrets.get(key); throw new RuntimeException("Key not found: " + key); } @Override public String getSecret(String key, int version) { - return secrets.get(key).get(version); - } - - public void put(String key, int version, String value) { - secrets.computeIfAbsent(key, k -> new HashMap<>()).put(version, value); + return getSecret(key); } public void put(String key, String value) { - put(key, 0, value); + secrets.put(key, value); } public void remove(String key) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 40115170b69..a099db5ebe8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; @@ -22,11 +22,6 @@ import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.MockReloadHandler; @@ -42,8 +37,7 @@ import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; -import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; -import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; +import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -52,14 +46,9 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import javax.security.auth.x500.X500Principal; import java.io.File; import java.io.IOException; -import java.math.BigInteger; -import java.security.KeyPair; -import java.security.cert.X509Certificate; import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -84,9 +73,6 @@ public class SessionPreparerTest { private static final File invalidTestApp = new File("src/test/apps/illegalApp"); private static final Version version123 = new Version(1, 2, 3); private static final Version version321 = new Version(3, 2, 1); - private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), - Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private MockCurator curator; @@ -245,37 +231,15 @@ public class SessionPreparerTest { var tlskey = "vespa.tlskeys.tenant1--app1"; var applicationId = applicationId("test"); var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); - - secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate)); - secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate())); - - prepare(new File("src/test/resources/deploy/hosted-app"), params); - - // Read from zk and verify cert and key are available - Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath) - .readEndpointCertificateMetadata(applicationId) - .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p)); - assertTrue(endpointCertificateSecrets.isPresent()); - assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); - assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); - } - - @Test - public void require_that_endpoint_certificate_metadata_is_written() throws IOException { - var applicationId = applicationId("test"); - var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build(); - secretStore.put("vespa.tlskeys.tenant1--app1-cert", 7, X509CertificateUtils.toPem(certificate)); - secretStore.put("vespa.tlskeys.tenant1--app1-key", 7, KeyUtils.toPem(keyPair.getPrivate())); + secretStore.put(tlskey+"-cert", "CERT"); + secretStore.put(tlskey+"-key", "KEY"); prepare(new File("src/test/resources/deploy/hosted-app"), params); // Read from zk and verify cert and key are available - Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath) - .readEndpointCertificateMetadata(applicationId) - .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p)); - - assertTrue(endpointCertificateSecrets.isPresent()); - assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); - assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); + Optional<TlsSecrets> tlsSecrets = new TlsSecretsKeys(curator, tenantPath, secretStore).readTlsSecretsKeyFromZookeeper(applicationId); + assertTrue(tlsSecrets.isPresent()); + assertEquals("KEY", tlsSecrets.get().key()); + assertEquals("CERT", tlsSecrets.get().certificate()); } @Test(expected = CertificateNotReadyException.class) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java deleted file mode 100644 index d71eab25ce3..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.model.api.EndpointCertificateMetadata; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.config.server.MockSecretStore; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Test; - -import javax.security.auth.x500.X500Principal; -import java.math.BigInteger; -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class EndpointCertificateMetadataStoreTest { - - private static final Path tenantPath = Path.createRoot(); - private static final Path endpointCertificateMetadataPath = Path.createRoot().append("tlsSecretsKeys").append("default:test:default"); - private static final ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(), - ApplicationName.from("test"), InstanceName.defaultName()); - - private MockCurator curator; - private MockSecretStore secretStore = new MockSecretStore(); - private EndpointCertificateMetadataStore endpointCertificateMetadataStore; - private EndpointCertificateRetriever endpointCertificateRetriever; - private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), - Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); - - @Before - public void setUp() { - curator = new MockCurator(); - endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); - endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); - - secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate)); - secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate())); - } - - @Test - public void reads_string_format() { - curator.set(endpointCertificateMetadataPath, ("\"vespa.tlskeys.tenant1--app1\"").getBytes()); - - // Read from zk and verify cert and key are available - var endpointCertificateSecrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId) - .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); - assertTrue(endpointCertificateSecrets.isPresent()); - assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); - assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); - } - - @Test - public void reads_object_format() { - curator.set(endpointCertificateMetadataPath, - "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}" - .getBytes()); - - // Read from zk and verify cert and key are available - var secrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId) - .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); - assertTrue(secrets.isPresent()); - assertTrue(secrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); - assertTrue(secrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); - } - - @Test - public void can_write_object_format() { - var endpointCertificateMetadata = new EndpointCertificateMetadata("key-name", "cert-name", 1); - - endpointCertificateMetadataStore.writeEndpointCertificateMetadata(applicationId, endpointCertificateMetadata); - - assertEquals("{\"keyName\":\"key-name\",\"certName\":\"cert-name\",\"version\":1}", - new String(curator.getData(endpointCertificateMetadataPath).orElseThrow())); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java new file mode 100644 index 00000000000..c71c7b8e040 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java @@ -0,0 +1,73 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.MockSecretStore; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TlsSecretsKeysTest { + + private static final Path tenantPath = Path.createRoot(); + private static final Path tlsSecretsKeysPath = Path.createRoot().append("tlsSecretsKeys").append("default:test:default"); + private static final String tlskey = "vespa.tlskeys.tenant1--app1"; + private static final ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(), + ApplicationName.from("test"), InstanceName.defaultName()); + + private MockCurator curator; + private MockSecretStore secretStore = new MockSecretStore(); + private TlsSecretsKeys tlsSecretsKeys; + + @Before + public void setUp() { + curator = new MockCurator(); + tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore); + secretStore.put(tlskey + "-cert", "CERT"); + secretStore.put(tlskey + "-key", "KEY"); + } + + @Test + public void reads_string_format() { + curator.set(tlsSecretsKeysPath, ('"' + tlskey + '"').getBytes()); + + // Read from zk and verify cert and key are available + var tlsSecrets = tlsSecretsKeys.readTlsSecretsKeyFromZookeeper(applicationId); + assertTrue(tlsSecrets.isPresent()); + assertEquals("KEY", tlsSecrets.get().key()); + assertEquals("CERT", tlsSecrets.get().certificate()); + } + + @Test + public void reads_object_format() { + curator.set(tlsSecretsKeysPath, + "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}" + .getBytes()); + + // Read from zk and verify cert and key are available + var tlsSecrets = tlsSecretsKeys.readTlsSecretsKeyFromZookeeper(applicationId); + assertTrue(tlsSecrets.isPresent()); + assertEquals("KEY", tlsSecrets.get().key()); + assertEquals("CERT", tlsSecrets.get().certificate()); + } + + @Test + public void can_write_object_format() { + var tlsSecretsMetadata = new TlsSecretsKeys.TlsSecretsMetadata(); + tlsSecretsMetadata.certName = "cert-name"; + tlsSecretsMetadata.keyName = "key-name"; + tlsSecretsMetadata.version = 1; + + tlsSecretsKeys.writeTlsSecretsMetadata(applicationId, tlsSecretsMetadata); + + assertEquals("{\"certName\":\"cert-name\",\"keyName\":\"key-name\",\"version\":1}", + new String(curator.getData(tlsSecretsKeysPath).get())); + } +} diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java index cefa8ab2f51..97b6cc344e1 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java @@ -19,16 +19,11 @@ import java.io.StringReader; import java.io.StringWriter; import java.io.UncheckedIOException; import java.security.GeneralSecurityException; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.security.Signature; -import java.security.SignatureException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Random; import static com.yahoo.security.Extension.SUBJECT_ALTERNATIVE_NAMES; import static java.util.stream.Collectors.toList; @@ -145,20 +140,4 @@ public class X509CertificateUtils { } } - public static boolean privateKeyMatchesPublicKey(PrivateKey privateKey, PublicKey publicKey) { - byte[] someRandomData = new byte[64]; - new Random().nextBytes(someRandomData); - - Signature signer = SignatureUtils.createSigner(privateKey); - Signature verifier = SignatureUtils.createVerifier(publicKey); - try { - signer.update(someRandomData); - verifier.update(someRandomData); - byte[] signature = signer.sign(); - return verifier.verify(signature); - } catch (SignatureException e) { - throw new RuntimeException(e); - } - } - } diff --git a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java index b4eca8328c1..76a93028efe 100644 --- a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java @@ -17,9 +17,7 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; /** * @author bjorncs @@ -73,18 +71,4 @@ public class X509CertificateUtilsTest { assertThat(sans.size(), is(1)); assertThat(sans.get(0), equalTo(san)); } - - @Test - public void verifies_matching_cert_and_key() { - KeyPair ecKeypairA = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - KeyPair ecKeypairB = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - KeyPair rsaKeypairA = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 1024); - KeyPair rsaKeypairB = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 1024); - - assertTrue(X509CertificateUtils.privateKeyMatchesPublicKey(ecKeypairA.getPrivate(), ecKeypairA.getPublic())); - assertTrue(X509CertificateUtils.privateKeyMatchesPublicKey(rsaKeypairA.getPrivate(), rsaKeypairA.getPublic())); - - assertFalse(X509CertificateUtils.privateKeyMatchesPublicKey(ecKeypairA.getPrivate(), ecKeypairB.getPublic())); - assertFalse(X509CertificateUtils.privateKeyMatchesPublicKey(rsaKeypairA.getPrivate(), rsaKeypairB.getPublic())); - } }
\ No newline at end of file |