diff options
author | Ola Aunrønning <olaa@yahooinc.com> | 2023-06-08 11:59:14 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-06-08 11:59:14 +0200 |
commit | ae28fde8ac675f69c9c5a0c61bda56bf1d70e42b (patch) | |
tree | ff703073a24f30e81efef1f1ca68f98c3cee3e3c | |
parent | 92e37cb2eb70d763eaebf6b26992bf6c74bcadfb (diff) | |
parent | 79de2d78433c11932357d0c244704b16fc87da21 (diff) |
Merge pull request #27331 from vespa-engine/olaa/dataplane-proxy-config
Dataplane proxy
11 files changed, 414 insertions, 24 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DataplaneProxy.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DataplaneProxy.java new file mode 100644 index 00000000000..fe7d9581e46 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DataplaneProxy.java @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.cloud.config.DataplaneProxyConfig; +import com.yahoo.container.jdisc.DataplaneProxyConfigurator; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +public class DataplaneProxy extends SimpleComponent implements DataplaneProxyConfig.Producer { + + private final Integer port; + private final String serverCertificate; + private final String serverKey; + + public DataplaneProxy(Integer port, String serverCertificate, String serverKey) { + super(DataplaneProxyConfigurator.class.getName()); + this.port = port; + this.serverCertificate = serverCertificate; + this.serverKey = serverKey; + } + + @Override + public void getConfig(DataplaneProxyConfig.Builder builder) { + builder.port(port); + builder.serverCertificate(serverCertificate); + builder.serverKey(serverKey); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredDirectSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/CloudSslProvider.java index adc1458ce85..5fa893e9599 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredDirectSslProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/CloudSslProvider.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container.http.ssl; import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ssl.impl.CloudSslContextProvider; import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; import java.util.Optional; @@ -14,9 +15,10 @@ import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; * @author mortent * @author andreer */ -public class ConfiguredDirectSslProvider extends SslProvider { +public class CloudSslProvider extends SslProvider { public static final String COMPONENT_ID_PREFIX = "configured-ssl-provider@"; - public static final String COMPONENT_CLASS = ConfiguredSslContextFactoryProvider.class.getName(); + public static final String MTLSONLY_COMPONENT_CLASS = ConfiguredSslContextFactoryProvider.class.getName(); + public static final String TOKEN_COMPONENT_CLASS = CloudSslContextProvider.class.getName(); private final String privateKey; private final String certificate; @@ -24,8 +26,8 @@ public class ConfiguredDirectSslProvider extends SslProvider { private final String caCertificate; private final ClientAuth.Enum clientAuthentication; - public ConfiguredDirectSslProvider(String servername, String privateKey, String certificate, String caCertificatePath, String caCertificate, ClientAuth.Enum clientAuthentication) { - super(COMPONENT_ID_PREFIX, servername, COMPONENT_CLASS, null); + public CloudSslProvider(String servername, String privateKey, String certificate, String caCertificatePath, String caCertificate, ClientAuth.Enum clientAuthentication, boolean enableTokenSupport) { + super(COMPONENT_ID_PREFIX, servername, componentClass(enableTokenSupport), null); this.privateKey = privateKey; this.certificate = certificate; this.caCertificatePath = caCertificatePath; @@ -33,6 +35,10 @@ public class ConfiguredDirectSslProvider extends SslProvider { this.clientAuthentication = clientAuthentication; } + private static String componentClass(boolean enableTokenSupport) { + return enableTokenSupport ? TOKEN_COMPONENT_CLASS : MTLSONLY_COMPONENT_CLASS; + } + @Override public void amendConnectorConfig(ConnectorConfig.Builder builder) { builder.ssl.enabled(true); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 72d2927f910..5bf348e5bb5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -33,9 +33,9 @@ public class HostedSslConnectorFactory extends ConnectorFactory { public static HostedSslConnectorFactory withProvidedCertificate( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth, Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port, - Duration endpointConnectionTtl) { - ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( - serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null, enforceHandshakeClientAuth); + Duration endpointConnectionTtl, boolean enableTokenSupport) { + CloudSslProvider sslProvider = createConfiguredDirectSslProvider( + serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null, enforceHandshakeClientAuth, enableTokenSupport); return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride, enableProxyProtocolMixedMode, port, endpointConnectionTtl); } @@ -46,9 +46,9 @@ public class HostedSslConnectorFactory extends ConnectorFactory { public static HostedSslConnectorFactory withProvidedCertificateAndTruststore( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates, Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port, - Duration endpointConnectionTtl) { - ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( - serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates, false); + Duration endpointConnectionTtl, boolean enableTokenSupport) { + CloudSslProvider sslProvider = createConfiguredDirectSslProvider( + serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates, false, enableTokenSupport); return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride, enableProxyProtocolMixedMode, port, endpointConnectionTtl); } @@ -74,16 +74,17 @@ public class HostedSslConnectorFactory extends ConnectorFactory { this.endpointConnectionTtl = endpointConnectionTtl; } - private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider( - String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificatesPath, String tlsCaCertificates, boolean enforceHandshakeClientAuth) { + private static CloudSslProvider createConfiguredDirectSslProvider( + String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificatesPath, String tlsCaCertificates, boolean enforceHandshakeClientAuth, boolean enableTokenSupport) { var clientAuthentication = enforceHandshakeClientAuth ? ClientAuth.Enum.NEED_AUTH : ClientAuth.Enum.WANT_AUTH; - return new ConfiguredDirectSslProvider( + return new CloudSslProvider( serverName, endpointCertificateSecrets.key(), endpointCertificateSecrets.certificate(), tlsCaCertificatesPath, tlsCaCertificates, - clientAuthentication); + clientAuthentication, + enableTokenSupport); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index e9517e1e64d..bcebf1a9fdd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -36,9 +36,11 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.ZoneEndpoint; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.jdisc.DataplaneProxyService; import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.http.server.jetty.DataplaneProxyCredentials; import com.yahoo.jdisc.http.server.jetty.VoidRequestLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.path.Path; @@ -67,6 +69,7 @@ import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.ContainerThreadpool; +import com.yahoo.vespa.model.container.DataplaneProxy; import com.yahoo.vespa.model.container.IdentityProvider; import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.SecretStore; @@ -465,13 +468,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private static void addCloudDataPlaneFilter(DeployState deployState, ApplicationContainerCluster cluster) { if (!deployState.isHosted() || !deployState.zone().system().isPublic()) return; + var dataplanePort = getDataplanePort(deployState); // Setup secure filter chain var secureChain = new HttpFilterChain("cloud-data-plane-secure", HttpFilterChain.Type.SYSTEM); secureChain.addInnerComponent(new CloudDataPlaneFilter(cluster, cluster.clientsLegacyMode())); cluster.getHttp().getFilterChains().add(secureChain); // Set cloud data plane filter as default request filter chain for data plane connector cluster.getHttp().getHttpServer().orElseThrow().getConnectorFactories().stream() - .filter(c -> c.getListenPort() == HOSTED_VESPA_DATAPLANE_PORT).findAny().orElseThrow() + .filter(c -> c.getListenPort() == dataplanePort).findAny().orElseThrow() .setDefaultRequestFilterChain(secureChain.getComponentId()); // Setup insecure filter chain @@ -566,6 +570,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Collection<String> tlsCiphersOverride = deployState.getProperties().tlsCiphersOverride(); boolean proxyProtocolMixedMode = deployState.getProperties().featureFlags().enableProxyProtocolMixedMode(); Duration endpointConnectionTtl = deployState.getProperties().endpointConnectionTtl(); + var port = getDataplanePort(deployState); if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); List<X509Certificate> clientCertificates = getClientCertificates(cluster); @@ -580,16 +585,39 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .map(clientAuth -> clientAuth == AccessControl.ClientAuthentication.need) .orElse(false); + // TODO (mortent): Implement token support in model + boolean enableTokenSupport = deployState.featureFlags().enableDataplaneProxy(); + + // Set up component to generate proxy cert if token support is enabled + if (enableTokenSupport) { + var tokenChain = new HttpFilterChain("cloud-data-plane-token", HttpFilterChain.Type.SYSTEM); + tokenChain.addInnerComponent(new Filter( + new ChainedComponentModel( + new BundleInstantiationSpecification( + new ComponentSpecification("com.yahoo.jdisc.http.filter.security.misc.BlockingRequestFilter"), + null, new ComponentSpecification("jdisc-security-filters")), + Dependencies.emptyDependencies()))); + + cluster.getHttp().getFilterChains().add(tokenChain); + + cluster.addSimpleComponent(DataplaneProxyCredentials.class); + cluster.addSimpleComponent(DataplaneProxyService.class); + var dataplaneProxy = new DataplaneProxy( + getDataplanePort(deployState), + endpointCertificateSecrets.certificate(), + endpointCertificateSecrets.key()); + cluster.addComponent(dataplaneProxy); + } connectorFactory = authorizeClient ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore( serverName, endpointCertificateSecrets, X509CertificateUtils.toPem(clientCertificates), - tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT, endpointConnectionTtl) + tlsCiphersOverride, proxyProtocolMixedMode, port, endpointConnectionTtl, enableTokenSupport) : HostedSslConnectorFactory.withProvidedCertificate( serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride, - proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT, endpointConnectionTtl); + proxyProtocolMixedMode, port, endpointConnectionTtl, enableTokenSupport); } else { connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore( - serverName, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT, + serverName, tlsCiphersOverride, proxyProtocolMixedMode, port, endpointConnectionTtl); } cluster.getHttp().getAccessControl().ifPresent(accessControl -> accessControl.configureHostedConnector(connectorFactory)); @@ -755,7 +783,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addUserHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { - OptionalInt portBindingOverride = isHostedTenantApplication(context) ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) : OptionalInt.empty(); + OptionalInt portBindingOverride = isHostedTenantApplication(context) ? OptionalInt.of(getDataplanePort(deployState)) : OptionalInt.empty(); for (Element component: XML.getChildren(spec, "handler")) { cluster.addComponent( new DomHandlerBuilder(cluster, portBindingOverride).build(deployState, cluster, component)); @@ -1046,7 +1074,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addSearchHandler(DeployState deployState, ApplicationContainerCluster cluster, Element searchElement, ConfigModelContext context) { BindingPattern bindingPattern = SearchHandler.DEFAULT_BINDING; if (isHostedTenantApplication(context) && deployState.featureFlags().useRestrictedDataPlaneBindings()) { - bindingPattern = SearchHandler.bindingPattern(Optional.of(Integer.toString(HOSTED_VESPA_DATAPLANE_PORT))); + bindingPattern = SearchHandler.bindingPattern(Optional.of(Integer.toString(getDataplanePort(deployState)))); } SearchHandler searchHandler = new SearchHandler(cluster, serverBindings(deployState, context, searchElement, bindingPattern), @@ -1067,7 +1095,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<BindingPattern> toBindingList(DeployState deployState, ConfigModelContext context, List<Element> bindingElements) { List<BindingPattern> result = new ArrayList<>(); - OptionalInt portOverride = isHostedTenantApplication(context) && deployState.featureFlags().useRestrictedDataPlaneBindings() ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) : OptionalInt.empty(); + OptionalInt portOverride = isHostedTenantApplication(context) && deployState.featureFlags().useRestrictedDataPlaneBindings() ? OptionalInt.of(getDataplanePort(deployState)) : OptionalInt.empty(); for (Element element: bindingElements) { String text = element.getTextContent().trim(); if (!text.isEmpty()) @@ -1090,7 +1118,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ContainerDocumentApi.HandlerOptions documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); Element ignoreUndefinedFields = XML.getChild(documentApiElement, "ignore-undefined-fields"); OptionalInt portBindingOverride = deployState.featureFlags().useRestrictedDataPlaneBindings() && isHostedTenantApplication(context) - ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) + ? OptionalInt.of(getDataplanePort(deployState)) : OptionalInt.empty(); return new ContainerDocumentApi(cluster, documentApiOptions, "true".equals(XML.getValue(ignoreUndefinedFields)), portBindingOverride); @@ -1351,4 +1379,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } + private static int getDataplanePort(DeployState deployState) { + return deployState.featureFlags().enableDataplaneProxy() ? 8443 : HOSTED_VESPA_DATAPLANE_PORT; + } + } diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index fd37244b825..496e92916ee 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -83,6 +83,9 @@ install_config_definition(onnx-models.def vespa.config.search.core.onnx-models.d vespa_generate_config(configdefinitions proton.def) install_config_definition(proton.def vespa.config.search.core.proton.def) vespa_generate_config(configdefinitions hwinfo.def) +vespa_generate_config(configdefinitions dataplane-proxy.def) +install_config_definition(dataplane-proxy.def cloud.config.dataplane-proxy.def) install_config_definition(hugging-face-embedder.def embedding.huggingface.hugging-face-embedder.def) install_config_definition(hugging-face-tokenizer.def language.huggingface.config.hugging-face-tokenizer.def) install_config_definition(bert-base-embedder.def embedding.bert-base-embedder.def) + diff --git a/configdefinitions/src/vespa/dataplane-proxy.def b/configdefinitions/src/vespa/dataplane-proxy.def new file mode 100644 index 00000000000..27f88583081 --- /dev/null +++ b/configdefinitions/src/vespa/dataplane-proxy.def @@ -0,0 +1,15 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +# The port Jdisc will be listening on +port int + +# Server certificate and key to be used when creating server socket +serverCertificate string +serverKey string + +# The mTLS endpoint SNI header to route to Jdisc using L4 +mTlsEndpoint string + +# The endpoint SNI header supporting token authentication +tokenEndpoint string diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java new file mode 100644 index 00000000000..46c840ad607 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java @@ -0,0 +1,70 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.X509CertificateWithKey; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.yolean.Exceptions; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.time.Duration; + +/** + * Generates temporary credentials to be used by a proxy for accessing Jdisc. + * Credentials are written to vespa_home/tmp/. + * + * @author mortent + */ +public class DataplaneProxyCredentials extends AbstractComponent { + + private final Path certificateFile; + private final Path keyFile; + + public DataplaneProxyCredentials() { + certificateFile = Paths.get(Defaults.getDefaults().underVespaHome("tmp/proxy_cert.pem")); + keyFile = Paths.get(Defaults.getDefaults().underVespaHome("tmp/proxy_key.pem")); + if (regenerateCredentials(certificateFile, keyFile)) { + X509CertificateWithKey selfSigned = X509CertificateUtils.createSelfSigned("cn=vespa dataplane proxy", Duration.ofDays(30)); + Exceptions.uncheck(() -> Files.writeString(certificateFile, X509CertificateUtils.toPem(selfSigned.certificate()))); + Exceptions.uncheck(() -> Files.writeString(keyFile, KeyUtils.toPem(selfSigned.privateKey()))); + } + } + + /* + * Returns true if credentials should be regenerated. + */ + private boolean regenerateCredentials(Path certificateFile, Path keyFile) { + if (!Files.exists(certificateFile) || !Files.exists(keyFile)) { + return true; + } + try { + X509Certificate x509Certificate = X509CertificateUtils.fromPem(Files.readString(certificateFile)); + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(Files.readString(keyFile)); + return !X509CertificateUtils.privateKeyMatchesPublicKey(privateKey, x509Certificate.getPublicKey()); + } catch (IOException e) { + // Some exception occured, assume credentials corrupted and requires a new pair. + return true; + } + } + + public Path certificateFile() { + return certificateFile; + } + + public Path keyFile() { + return keyFile; + } + + @Override + public void deconstruct() { + super.deconstruct(); + } + +} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/CloudSslContextProvider.java b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/CloudSslContextProvider.java new file mode 100644 index 00000000000..cdfd4aa938e --- /dev/null +++ b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/CloudSslContextProvider.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.server.jetty.DataplaneProxyCredentials; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Optional; + +/** + * Used to enable token based endpoints in Cloud. Amends trust store to allow proxy. + * + * @author mortent + */ +public class CloudSslContextProvider extends ConfiguredSslContextFactoryProvider { + + private final DataplaneProxyCredentials dataplaneProxyCredentials; + + public CloudSslContextProvider(ConnectorConfig connectorConfig, DataplaneProxyCredentials dataplaneProxyCredentials) { + super(connectorConfig); + this.dataplaneProxyCredentials = dataplaneProxyCredentials; + } + + @Override + Optional<String> getCaCertificates(ConnectorConfig.Ssl sslConfig) { + String proxyCert; + try { + proxyCert = Files.readString(dataplaneProxyCredentials.certificateFile(), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new IllegalArgumentException("Dataplane proxy certificate not available", e); + } + if (!sslConfig.caCertificate().isBlank()) { + return Optional.of(sslConfig.caCertificate() + "\n" + proxyCert); + } else if (!sslConfig.caCertificateFile().isBlank()) { + return Optional.of(readToString(sslConfig.caCertificateFile()) + "\n" + proxyCert); + } else { + return Optional.of(proxyCert); + } + } +} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java index 27c5aff22a9..b99bc007b32 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java @@ -110,7 +110,7 @@ public class ConfiguredSslContextFactoryProvider implements SslProvider { private static boolean hasBoth(String a, String b) { return !a.isBlank() && !b.isBlank(); } private static boolean hasNeither(String a, String b) { return a.isBlank() && b.isBlank(); } - private static Optional<String> getCaCertificates(ConnectorConfig.Ssl sslConfig) { + Optional<String> getCaCertificates(ConnectorConfig.Ssl sslConfig) { if (!sslConfig.caCertificate().isBlank()) { return Optional.of(sslConfig.caCertificate()); } else if (!sslConfig.caCertificateFile().isBlank()) { @@ -130,7 +130,7 @@ public class ConfiguredSslContextFactoryProvider implements SslProvider { return readToString(config.certificateFile()); } - private static String readToString(String filename) { + static String readToString(String filename) { try { return Files.readString(Paths.get(filename), StandardCharsets.UTF_8); } catch (IOException e) { diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyConfigurator.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyConfigurator.java new file mode 100644 index 00000000000..340c0035052 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyConfigurator.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.cloud.config.DataplaneProxyConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.jdisc.http.server.jetty.DataplaneProxyCredentials; + +/** + * Reconfigurable component for supporting data plane proxy. Configures the {@code DataplaneProxyService} by calling {@code DataplaneProxyService#init} + * + * @author mortent + */ +public class DataplaneProxyConfigurator extends AbstractComponent { + + public DataplaneProxyConfigurator(DataplaneProxyConfig config, DataplaneProxyService dataplaneProxyService, DataplaneProxyCredentials credentialsProvider) { + dataplaneProxyService.reconfigure(config, credentialsProvider); + } + + @Override + public void deconstruct() { + super.deconstruct(); + } +} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java new file mode 100644 index 00000000000..230d017c584 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java @@ -0,0 +1,170 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.cloud.config.DataplaneProxyConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.jdisc.http.server.jetty.DataplaneProxyCredentials; + +import javax.inject.Inject; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + +/** + * Configures a data plane proxy. Currently using Nginx. + * + * @author mortent + */ +public class DataplaneProxyService extends AbstractComponent { + + private static final String PREFIX = "/opt/vespa"; + private static final Path CONFIG_TEMPLATE = Paths.get(PREFIX, "conf/nginx/nginx.conf.template"); + + private static final Path clientCertificateFile = Paths.get(PREFIX, "conf/nginx/client_cert.pem"); + private static final Path clientKeyFile = Paths.get(PREFIX, "conf/nginx/client_key.pem"); + private static final Path serverCertificateFile = Paths.get(PREFIX, "conf/nginx/server_cert.pem"); + private static final Path serverKeyFile = Paths.get(PREFIX, "conf/nginx/server_key.pem"); + + private static final Path nginxConf = Paths.get(PREFIX, "conf/nginx/nginx.conf"); + + private boolean started; + + @Inject + public DataplaneProxyService() { + this.started = false; + } + + public void reconfigure(DataplaneProxyConfig config, DataplaneProxyCredentials credentialsProvider) { + try { + String serverCert = config.serverCertificate(); + String serverKey = config.serverKey(); + + boolean configChanged = false; + configChanged |= writeFile(serverCertificateFile, serverCert); + configChanged |= writeFile(serverKeyFile, serverKey); + configChanged |= writeFile(nginxConf, + nginxConfig( + credentialsProvider.certificateFile(), + credentialsProvider.keyFile(), + serverCertificateFile, + serverKeyFile, + URI.create(config.mTlsEndpoint()), + URI.create(config.tokenEndpoint()), + config.port(), + PREFIX + )); + if (!started) { + startNginx(); + started = true; + } else if (configChanged){ + reloadNginx(); + } + } catch (IOException e) { + throw new RuntimeException("Error reconfiguring data plane proxy", e); + } + } + + private void startNginx() { + try { + Process startCommand = new ProcessBuilder().command( + "nginx", + "-c", nginxConf.toString() + ).start(); + int exitCode = startCommand.waitFor(); + if (exitCode != 0) { + throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode)); + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not start nginx", e); + } + } + + private void reloadNginx() { + try { + Process reloadCommand = new ProcessBuilder().command( + "nginx", + "-s", "reload" + ).start(); + int exitCode = reloadCommand.waitFor(); + if (exitCode != 0) { + throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode)); + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not start nginx", e); + } + } + + private void stopNginx() { + try { + Process stopCommand = new ProcessBuilder().command( + "nginx", + "-s", "reload" + ).start(); + int exitCode = stopCommand.waitFor(); + if (exitCode != 0) { + throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode)); + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not start nginx", e); + } + } + + @Override + public void deconstruct() { + super.deconstruct(); + stopNginx(); + } + + /* + * Writes a file to disk + * return true if file was changed, false if no changes + */ + private boolean writeFile(Path file, String contents) throws IOException { + Path tempPath = Paths.get(file.toFile().getAbsolutePath() + ".new"); + Files.createDirectories(tempPath.getParent()); + Files.writeString(tempPath, contents); + + if (!Files.exists(file) || Files.mismatch(tempPath, file) > 0) { + Files.move(tempPath, file, StandardCopyOption.REPLACE_EXISTING); + return true; + } else { + Files.delete(tempPath); + return false; + } + } + + static String nginxConfig( + Path clientCert, + Path clientKey, + Path serverCert, + Path serverKey, + URI mTlsEndpoint, + URI tokenEndpoint, + int vespaPort, + String prefix) { + + try { + String nginxTemplate = Files.readString(CONFIG_TEMPLATE); + nginxTemplate = replace(nginxTemplate, "client_cert", clientCert.toString()); + nginxTemplate = replace(nginxTemplate, "client_key", clientKey.toString()); + nginxTemplate = replace(nginxTemplate, "server_cert", serverCert.toString()); + nginxTemplate = replace(nginxTemplate, "server_key", serverKey.toString()); + nginxTemplate = replace(nginxTemplate, "mtls_endpoint", mTlsEndpoint.getHost()); + nginxTemplate = replace(nginxTemplate, "token_endpoint", tokenEndpoint.getHost()); + nginxTemplate = replace(nginxTemplate, "vespa_port", Integer.toString(vespaPort)); + nginxTemplate = replace(nginxTemplate, "prefix", prefix); + + // TODO: verify that all template vars have been expanded + return nginxTemplate; + } catch (IOException e) { + throw new IllegalArgumentException("Could not create data plane proxy configuration", e); + } + } + + private static String replace(String template, String key, String value) { + return template.replaceAll("\\$\\{%s\\}".formatted(key), value); + } +} |