Merge pull request #21000 from vespa-engine/bjorncs/feature-flag-pre-shutdown

Bjorncs/feature flag pre shutdown

7 files changed, 31 insertions, 3 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 952ff090693..fb878f2c75f 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -110,6 +110,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsDecrementFactor() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; }
+        @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return false; }
     }
 
     /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
index 43cb896c09b..54b9a4511b0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
@@ -11,6 +11,10 @@ import com.yahoo.config.provision.NodeResources;
 import com.yahoo.search.config.QrStartConfig;
 import com.yahoo.vespa.model.container.component.SimpleComponent;
 
+import java.util.Optional;
+
+import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+
 /**
  * A container that is typically used by container clusters set up from the user application.
  *
@@ -25,6 +29,7 @@ public final class ApplicationContainer extends Container implements
     private final boolean isHostedVespa;
     private final boolean enableServerOcspStapling;
     private final boolean useQrserverServiceName;
+    private final boolean enablePreshutdownCommand;
 
     public ApplicationContainer(AbstractConfigProducer<?> parent, String name, int index, DeployState deployState) {
         this(parent, name, false, index, deployState);
@@ -35,6 +40,7 @@ public final class ApplicationContainer extends Container implements
         this.isHostedVespa = deployState.isHosted();
         this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling();
         this.useQrserverServiceName = deployState.featureFlags().useQrserverServiceName();
+        this.enablePreshutdownCommand = deployState.featureFlags().enableJdiscPreshutdownCommand();
 
         addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder"));
         addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider"));
@@ -101,4 +107,12 @@ public final class ApplicationContainer extends Container implements
         return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container);
     }
 
+    @Override
+    public Optional<String> getPreShutdownCommand() {
+        if (!enablePreshutdownCommand) return Optional.empty();
+        int preshutdownTimeoutSeconds = 360;
+        int rpcTimeoutSeconds = preshutdownTimeoutSeconds + 10;
+        String rpcParams = "-t " + rpcTimeoutSeconds + " tcp/localhost:" + getRpcPort() + " prepareStop d:" + preshutdownTimeoutSeconds;
+        return Optional.of(getDefaults().underVespaHome("bin/vespa-rpc-invoke") + " " + rpcParams);
+    }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
index 5c5b065ac29..2e5126dcc44 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -281,7 +281,7 @@ public abstract class Container extends AbstractService implements
     }
 
     protected int allocatedRpcPort = 0;
-    private int getRpcPort() {
+    protected int getRpcPort() {
         return allocatedRpcPort;
     }
     protected int numRpcPorts() { return rpcServerEnabled() ? 1 : 0; }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
index 7b91b4b3244..e397e6ab399 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
@@ -22,6 +22,7 @@ import java.util.List;
 public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer {
 
     private final ContainerCluster<?> cluster;
+    private final boolean enablePreshutdownCommand;
     private volatile boolean isHostedVespa;
     private final List<ConnectorFactory> connectorFactories = new ArrayList<>();
     private final List<String> ignoredUserAgentsList = new ArrayList<>();
@@ -36,6 +37,7 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro
         for (String agent : deployState.featureFlags().ignoredHttpUserAgents()) {
             addIgnoredUserAgent(agent);
         }
+        this.enablePreshutdownCommand = deployState.featureFlags().enableJdiscPreshutdownCommand();
     }
 
     public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; }
@@ -75,6 +77,9 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro
                     .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp")));
         }
         configureJettyThreadpool(builder);
+        if (enablePreshutdownCommand) {
+            builder.stopTimeout(300);
+        }
     }
 
     private void configureJettyThreadpool(ServerConfig.Builder builder) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index bc0f66d933c..056ca4b6c0d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -205,6 +205,7 @@ public class ModelContextImpl implements ModelContext {
         private final double persistenceThrottlingWsDecrementFactor;
         private final double persistenceThrottlingWsBackoff;
         private final boolean useQrserverServiceName;
+        private final boolean enableJdiscPreshutdownCommand;
 
         public FeatureFlags(FlagSource source, ApplicationId appId) {
             this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT);
@@ -248,6 +249,7 @@ public class ModelContextImpl implements ModelContext {
             this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR);
             this.persistenceThrottlingWsBackoff = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF);
             this.useQrserverServiceName =  flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME);
+            this.enableJdiscPreshutdownCommand =  flagValue(source, appId, Flags.ENABLE_JDISC_PRESHUTDOWN_COMMAND);
         }
 
         @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
@@ -293,6 +295,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; }
         @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; }
         @Override public boolean useQrserverServiceName() { return useQrserverServiceName; }
+        @Override public boolean enableJdiscPreshutdownCommand() { return enableJdiscPreshutdownCommand; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
index 606e2bf6cb4..70f708bdd0a 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
@@ -392,8 +392,6 @@ public final class ConfiguredApplication implements Application {
         } catch (Exception e) {
             request.setError(ErrorCode.METHOD_FAILED, e.getMessage());
             throw e;
-        } finally {
-            request.returnRequest();
         }
     }
 
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index e2c7d376a0d..9b7dcea6c18 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -395,6 +395,13 @@ public class Flags {
             "Takes effect at redeployment",
             ZONE_ID, APPLICATION_ID);
 
+    public static final UnboundBooleanFlag ENABLE_JDISC_PRESHUTDOWN_COMMAND = defineFeatureFlag(
+            "enable-jdisc-preshutdown-command", false,
+            List.of("bjorncs", "baldersheim"), "2022-01-31", "2022-05-31",
+            "Enable pre-shutdown command for jdisc",
+            "Takes effect at redeployment",
+            APPLICATION_ID, HOSTNAME, TENANT_ID);
+
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
                                                        String createdAt, String expiresAt, String description,