Merge pull request #26867 from vespa-engine/bjorncs/feature-flag-endpoint-conn-ttl

Add feature flag to control connection TTL

5 files changed, 42 insertions, 19 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 7f2dd4b6acd..ee7ca64166d 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -20,6 +20,7 @@ import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 import java.net.URI;
 import java.security.cert.X509Certificate;
+import java.time.Duration;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -185,6 +186,8 @@ public interface ModelContext {
 
         default boolean allowUserFilters() { return true; }
 
+        default Duration endpointConnectionTtl() { return Duration.ZERO; }
+
     }
 
     @Retention(RetentionPolicy.RUNTIME)
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
index 4aa0386bd27..72d2927f910 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
@@ -10,7 +10,6 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory;
 import java.time.Duration;
 import java.util.Collection;
 import java.util.List;
-import java.util.stream.Collectors;
 
 /**
  * Component specification for {@link com.yahoo.jdisc.http.server.jetty.ConnectorFactory} with hosted specific configuration.
@@ -26,16 +25,19 @@ public class HostedSslConnectorFactory extends ConnectorFactory {
     private final boolean enforceHandshakeClientAuth;
     private final Collection<String> tlsCiphersOverride;
     private final boolean enableProxyProtocolMixedMode;
+    private final Duration endpointConnectionTtl;
 
     /**
      * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore.
      */
     public static HostedSslConnectorFactory withProvidedCertificate(
             String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth,
-            Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port) {
+            Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port,
+            Duration endpointConnectionTtl) {
         ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider(
                 serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null, enforceHandshakeClientAuth);
-        return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride, enableProxyProtocolMixedMode, port);
+        return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride,
+                                             enableProxyProtocolMixedMode, port, endpointConnectionTtl);
     }
 
     /**
@@ -43,28 +45,33 @@ public class HostedSslConnectorFactory extends ConnectorFactory {
      */
     public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(
             String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates,
-            Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port) {
+            Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port,
+            Duration endpointConnectionTtl) {
         ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider(
                 serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates, false);
-        return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride, enableProxyProtocolMixedMode, port);
+        return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride, enableProxyProtocolMixedMode,
+                                             port, endpointConnectionTtl);
     }
 
     /**
      * Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration).
      */
     public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName, Collection<String> tlsCiphersOverride,
-                                                                                boolean enableProxyProtocolMixedMode, int port) {
-        return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, tlsCiphersOverride, enableProxyProtocolMixedMode, port);
+                                                                                boolean enableProxyProtocolMixedMode, int port,
+                                                                                Duration endpointConnectionTtl) {
+        return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, tlsCiphersOverride,
+                                             enableProxyProtocolMixedMode, port, endpointConnectionTtl);
     }
 
     private HostedSslConnectorFactory(SslProvider sslProvider, boolean enforceClientAuth,
                                       boolean enforceHandshakeClientAuth, Collection<String> tlsCiphersOverride,
-                                      boolean enableProxyProtocolMixedMode, int port) {
+                                      boolean enableProxyProtocolMixedMode, int port, Duration endpointConnectionTtl) {
         super(new Builder("tls"+port, port).sslProvider(sslProvider));
         this.enforceClientAuth = enforceClientAuth;
         this.enforceHandshakeClientAuth = enforceHandshakeClientAuth;
         this.tlsCiphersOverride = tlsCiphersOverride;
         this.enableProxyProtocolMixedMode = enableProxyProtocolMixedMode;
+        this.endpointConnectionTtl = endpointConnectionTtl;
     }
 
     private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider(
@@ -100,6 +107,6 @@ public class HostedSslConnectorFactory extends ConnectorFactory {
         connectorBuilder
                 .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder().enabled(true).mixedMode(enableProxyProtocolMixedMode))
                 .idleTimeout(Duration.ofSeconds(30).toSeconds())
-                .maxConnectionLife(Duration.ofSeconds(45).toSeconds());
+                .maxConnectionLife(endpointConnectionTtl != null ? endpointConnectionTtl.toSeconds() : 0);
     }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index f74c218a906..cf6ca4bed3d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -2,8 +2,6 @@
 package com.yahoo.vespa.model.container.xml;
 
 import com.yahoo.component.ComponentId;
-import com.yahoo.config.provision.ClusterInfo;
-import com.yahoo.config.provision.IntRange;
 import com.yahoo.component.ComponentSpecification;
 import com.yahoo.component.Version;
 import com.yahoo.component.chain.dependencies.Dependencies;
@@ -14,8 +12,6 @@ import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.config.provision.ZoneEndpoint;
-import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader;
 import com.yahoo.config.model.ConfigModelContext;
 import com.yahoo.config.model.api.ApplicationClusterEndpoint;
 import com.yahoo.config.model.api.ConfigServerSpec;
@@ -32,13 +28,12 @@ import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterMembership;
-import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.config.provision.ZoneEndpoint;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.container.bundle.BundleInstantiationSpecification;
 import com.yahoo.container.logging.FileConnectionLog;
@@ -91,10 +86,10 @@ import com.yahoo.vespa.model.container.http.Filter;
 import com.yahoo.vespa.model.container.http.FilterBinding;
 import com.yahoo.vespa.model.container.http.FilterChains;
 import com.yahoo.vespa.model.container.http.Http;
+import com.yahoo.vespa.model.container.http.HttpFilterChain;
 import com.yahoo.vespa.model.container.http.JettyHttpServer;
 import com.yahoo.vespa.model.container.http.ssl.HostedSslConnectorFactory;
 import com.yahoo.vespa.model.container.http.xml.HttpBuilder;
-import com.yahoo.vespa.model.container.http.HttpFilterChain;
 import com.yahoo.vespa.model.container.processing.ProcessingChains;
 import com.yahoo.vespa.model.container.search.ContainerSearch;
 import com.yahoo.vespa.model.container.search.PageTemplates;
@@ -108,6 +103,7 @@ import java.io.IOException;
 import java.io.Reader;
 import java.net.URI;
 import java.security.cert.X509Certificate;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -567,6 +563,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         HostedSslConnectorFactory connectorFactory;
         Collection<String> tlsCiphersOverride = deployState.getProperties().tlsCiphersOverride();
         boolean proxyProtocolMixedMode = deployState.getProperties().featureFlags().enableProxyProtocolMixedMode();
+        Duration endpointConnectionTtl = deployState.getProperties().endpointConnectionTtl();
         if (deployState.endpointCertificateSecrets().isPresent()) {
             boolean authorizeClient = deployState.zone().system().isPublic();
             List<X509Certificate> clientCertificates = getClientCertificates(cluster);
@@ -583,11 +580,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
 
             connectorFactory = authorizeClient
                     ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(
-                    serverName, endpointCertificateSecrets, X509CertificateUtils.toPem(clientCertificates), tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT)
+                    serverName, endpointCertificateSecrets, X509CertificateUtils.toPem(clientCertificates),
+                    tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT, endpointConnectionTtl)
                     : HostedSslConnectorFactory.withProvidedCertificate(
-                    serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT);
+                    serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride,
+                    proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT, endpointConnectionTtl);
         } else {
-            connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT);
+            connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(
+                    serverName, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT,
+                    endpointConnectionTtl);
         }
         cluster.getHttp().getAccessControl().ifPresent(accessControl -> accessControl.configureHostedConnector(connectorFactory));
         server.addConnector(connectorFactory);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 62431ce4c06..b443a0e64b0 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -36,6 +36,7 @@ import com.yahoo.vespa.flags.UnboundFlag;
 import java.io.File;
 import java.net.URI;
 import java.security.cert.X509Certificate;
+import java.time.Duration;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -379,6 +380,7 @@ public class ModelContextImpl implements ModelContext {
         private final List<String> environmentVariables;
         private final Optional<CloudAccount> cloudAccount;
         private final boolean allowUserFilters;
+        private final Duration endpointConnectionTtl;
 
         public Properties(ApplicationId applicationId,
                           Version modelVersion,
@@ -425,6 +427,9 @@ public class ModelContextImpl implements ModelContext {
             this.cloudAccount = cloudAccount;
             this.allowUserFilters = PermanentFlags.ALLOW_USER_FILTERS.bindTo(flagSource)
                     .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+            this.endpointConnectionTtl = Duration.ofSeconds(
+                    PermanentFlags.ENDPOINT_CONNECTION_TTL.bindTo(flagSource)
+                            .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value());
         }
 
         @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; }
@@ -516,6 +521,7 @@ public class ModelContextImpl implements ModelContext {
 
         @Override public boolean allowUserFilters() { return allowUserFilters; }
 
+        @Override public Duration endpointConnectionTtl() { return endpointConnectionTtl; }
     }
 
 }
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
index f5fbc26e099..e80348261ef 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
@@ -354,6 +354,12 @@ public class PermanentFlags {
             ZONE_ID, APPLICATION_ID
     );
 
+    public static final UnboundIntFlag ENDPOINT_CONNECTION_TTL = defineIntFlag(
+            "endpoint-connection-ttl", 45,
+            "Time to live for connections to endpoints in seconds",
+            "Takes effect on next redeployment",
+            APPLICATION_ID);
+
     private PermanentFlags() {}
 
     private static UnboundBooleanFlag defineFeatureFlag(