From 8c53320a1bc10b81b761679c69ce1fac4b85fc3c Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Wed, 26 Apr 2023 15:13:06 +0200 Subject: Add feature flag to control connection TTL --- .../com/yahoo/config/model/api/ModelContext.java | 3 +++ .../http/ssl/HostedSslConnectorFactory.java | 25 ++++++++++++++-------- .../model/container/xml/ContainerModelBuilder.java | 21 +++++++++--------- .../config/server/deploy/ModelContextImpl.java | 6 ++++++ .../java/com/yahoo/vespa/flags/PermanentFlags.java | 6 ++++++ 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 7f2dd4b6acd..ee7ca64166d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -20,6 +20,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.net.URI; import java.security.cert.X509Certificate; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.Set; @@ -185,6 +186,8 @@ public interface ModelContext { default boolean allowUserFilters() { return true; } + default Duration endpointConnectionTtl() { return Duration.ZERO; } + } @Retention(RetentionPolicy.RUNTIME) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 4aa0386bd27..72d2927f910 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory; import java.time.Duration; import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; /** * Component specification for {@link com.yahoo.jdisc.http.server.jetty.ConnectorFactory} with hosted specific configuration. @@ -26,16 +25,19 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private final boolean enforceHandshakeClientAuth; private final Collection tlsCiphersOverride; private final boolean enableProxyProtocolMixedMode; + private final Duration endpointConnectionTtl; /** * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore. */ public static HostedSslConnectorFactory withProvidedCertificate( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth, - Collection tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port) { + Collection tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port, + Duration endpointConnectionTtl) { ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null, enforceHandshakeClientAuth); - return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride, enableProxyProtocolMixedMode, port); + return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride, + enableProxyProtocolMixedMode, port, endpointConnectionTtl); } /** @@ -43,28 +45,33 @@ public class HostedSslConnectorFactory extends ConnectorFactory { */ public static HostedSslConnectorFactory withProvidedCertificateAndTruststore( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates, - Collection tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port) { + Collection tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port, + Duration endpointConnectionTtl) { ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates, false); - return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride, enableProxyProtocolMixedMode, port); + return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride, enableProxyProtocolMixedMode, + port, endpointConnectionTtl); } /** * Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration). */ public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName, Collection tlsCiphersOverride, - boolean enableProxyProtocolMixedMode, int port) { - return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, tlsCiphersOverride, enableProxyProtocolMixedMode, port); + boolean enableProxyProtocolMixedMode, int port, + Duration endpointConnectionTtl) { + return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, tlsCiphersOverride, + enableProxyProtocolMixedMode, port, endpointConnectionTtl); } private HostedSslConnectorFactory(SslProvider sslProvider, boolean enforceClientAuth, boolean enforceHandshakeClientAuth, Collection tlsCiphersOverride, - boolean enableProxyProtocolMixedMode, int port) { + boolean enableProxyProtocolMixedMode, int port, Duration endpointConnectionTtl) { super(new Builder("tls"+port, port).sslProvider(sslProvider)); this.enforceClientAuth = enforceClientAuth; this.enforceHandshakeClientAuth = enforceHandshakeClientAuth; this.tlsCiphersOverride = tlsCiphersOverride; this.enableProxyProtocolMixedMode = enableProxyProtocolMixedMode; + this.endpointConnectionTtl = endpointConnectionTtl; } private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider( @@ -100,6 +107,6 @@ public class HostedSslConnectorFactory extends ConnectorFactory { connectorBuilder .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder().enabled(true).mixedMode(enableProxyProtocolMixedMode)) .idleTimeout(Duration.ofSeconds(30).toSeconds()) - .maxConnectionLife(Duration.ofSeconds(45).toSeconds()); + .maxConnectionLife(endpointConnectionTtl != null ? endpointConnectionTtl.toSeconds() : 0); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index f74c218a906..cf6ca4bed3d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.component.ComponentId; -import com.yahoo.config.provision.ClusterInfo; -import com.yahoo.config.provision.IntRange; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; import com.yahoo.component.chain.dependencies.Dependencies; @@ -14,8 +12,6 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ZoneEndpoint; -import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ConfigServerSpec; @@ -32,13 +28,12 @@ import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneEndpoint; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.logging.FileConnectionLog; @@ -91,10 +86,10 @@ import com.yahoo.vespa.model.container.http.Filter; import com.yahoo.vespa.model.container.http.FilterBinding; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; +import com.yahoo.vespa.model.container.http.HttpFilterChain; import com.yahoo.vespa.model.container.http.JettyHttpServer; import com.yahoo.vespa.model.container.http.ssl.HostedSslConnectorFactory; import com.yahoo.vespa.model.container.http.xml.HttpBuilder; -import com.yahoo.vespa.model.container.http.HttpFilterChain; import com.yahoo.vespa.model.container.processing.ProcessingChains; import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.PageTemplates; @@ -108,6 +103,7 @@ import java.io.IOException; import java.io.Reader; import java.net.URI; import java.security.cert.X509Certificate; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -567,6 +563,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { HostedSslConnectorFactory connectorFactory; Collection tlsCiphersOverride = deployState.getProperties().tlsCiphersOverride(); boolean proxyProtocolMixedMode = deployState.getProperties().featureFlags().enableProxyProtocolMixedMode(); + Duration endpointConnectionTtl = deployState.getProperties().endpointConnectionTtl(); if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); List clientCertificates = getClientCertificates(cluster); @@ -583,11 +580,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder { connectorFactory = authorizeClient ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore( - serverName, endpointCertificateSecrets, X509CertificateUtils.toPem(clientCertificates), tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT) + serverName, endpointCertificateSecrets, X509CertificateUtils.toPem(clientCertificates), + tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT, endpointConnectionTtl) : HostedSslConnectorFactory.withProvidedCertificate( - serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT); + serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride, + proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT, endpointConnectionTtl); } else { - connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT); + connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore( + serverName, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT, + endpointConnectionTtl); } cluster.getHttp().getAccessControl().ifPresent(accessControl -> accessControl.configureHostedConnector(connectorFactory)); server.addConnector(connectorFactory); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 62431ce4c06..b443a0e64b0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -36,6 +36,7 @@ import com.yahoo.vespa.flags.UnboundFlag; import java.io.File; import java.net.URI; import java.security.cert.X509Certificate; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.Set; @@ -379,6 +380,7 @@ public class ModelContextImpl implements ModelContext { private final List environmentVariables; private final Optional cloudAccount; private final boolean allowUserFilters; + private final Duration endpointConnectionTtl; public Properties(ApplicationId applicationId, Version modelVersion, @@ -425,6 +427,9 @@ public class ModelContextImpl implements ModelContext { this.cloudAccount = cloudAccount; this.allowUserFilters = PermanentFlags.ALLOW_USER_FILTERS.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.endpointConnectionTtl = Duration.ofSeconds( + PermanentFlags.ENDPOINT_CONNECTION_TTL.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value()); } @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; } @@ -516,6 +521,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean allowUserFilters() { return allowUserFilters; } + @Override public Duration endpointConnectionTtl() { return endpointConnectionTtl; } } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index f5fbc26e099..e80348261ef 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -354,6 +354,12 @@ public class PermanentFlags { ZONE_ID, APPLICATION_ID ); + public static final UnboundIntFlag ENDPOINT_CONNECTION_TTL = defineIntFlag( + "endpoint-connection-ttl", 45, + "Time to live for connections to endpoints in seconds", + "Takes effect on next redeployment", + APPLICATION_ID); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( -- cgit v1.2.3