Merge pull request #20627 from vespa-engine/arnej/feature-flag-for-user-agents

Arnej/feature flag for user agents

7 files changed, 21 insertions, 7 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 9fef9b4615d..a0dba36fef5 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -113,6 +113,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; }
         @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; }
         @ModelFeatureFlag(owners = {"baldersheim"}) default double tlsSizeFraction() { throw new UnsupportedOperationException("TODO specify default value"); }
+        @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); }
         @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
index 3ac90673bb6..5c5b065ac29 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -94,7 +94,7 @@ public abstract class Container extends AbstractService implements
         this.index = index;
         dumpHeapOnShutdownTimeout = deployState.featureFlags().containerDumpHeapOnShutdownTimeout();
         shutdownTimeoutS = deployState.featureFlags().containerShutdownTimeout();
-        this.defaultHttpServer = new JettyHttpServer("DefaultHttpServer", containerClusterOrNull(parent), deployState.isHosted());
+        this.defaultHttpServer = new JettyHttpServer("DefaultHttpServer", containerClusterOrNull(parent), deployState);
         if (getHttp() == null) {
             addChild(defaultHttpServer);
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
index d84884665ef..7b91b4b3244 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.container.http;
 
+import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.component.ComponentId;
 import com.yahoo.component.ComponentSpecification;
 import com.yahoo.container.bundle.BundleInstantiationSpecification;
@@ -25,13 +26,16 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro
     private final List<ConnectorFactory> connectorFactories = new ArrayList<>();
     private final List<String> ignoredUserAgentsList = new ArrayList<>();
 
-    public JettyHttpServer(String componentId, ContainerCluster<?> cluster, boolean isHostedVespa) {
+    public JettyHttpServer(String componentId, ContainerCluster<?> cluster, DeployState deployState) {
         super(new ComponentModel(componentId, com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName(), null));
-        this.isHostedVespa = isHostedVespa;
+        this.isHostedVespa = deployState.isHosted();
         this.cluster = cluster;
         final FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(componentId);
         addChild(filterBindingsProviderComponent);
         inject(filterBindingsProviderComponent);
+        for (String agent : deployState.featureFlags().ignoredHttpUserAgents()) {
+            addIgnoredUserAgent(agent);
+        }
     }
 
     public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java
index 07f4334b542..1518683908b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java
@@ -24,7 +24,7 @@ public class JettyHttpServerBuilder extends VespaDomBuilder.DomConfigProducerBui
 
     @Override
     protected JettyHttpServer doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element http) {
-        JettyHttpServer jettyHttpServer = new JettyHttpServer("jdisc-jetty", cluster, deployState.isHosted());
+        JettyHttpServer jettyHttpServer = new JettyHttpServer("jdisc-jetty", cluster, deployState);
         for (Element serverSpec: XML.getChildren(http, "server")) {
             ConnectorFactory connectorFactory = new JettyConnectorBuilder().build(deployState, ancestor, serverSpec);
             jettyHttpServer.addConnector(connectorFactory);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 2da68262fe7..d5c64689648 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -426,7 +426,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
             cluster.setHttp(buildHttp(deployState, cluster, httpElement));
         }
         if (isHostedTenantApplication(context)) {
-            addHostedImplicitHttpIfNotPresent(cluster);
+            addHostedImplicitHttpIfNotPresent(deployState, cluster);
             addHostedImplicitAccessControlIfNotPresent(deployState, cluster);
             addDefaultConnectorHostedFilterBinding(cluster);
             addAdditionalHostedConnector(deployState, cluster, context);
@@ -488,13 +488,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         return deployState.isHosted() && context.getApplicationType() == ApplicationType.DEFAULT && !isTesterApplication;
     }
 
-    private static void addHostedImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) {
+    private static void addHostedImplicitHttpIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) {
         if (cluster.getHttp() == null) {
             cluster.setHttp(new Http(new FilterChains(cluster)));
         }
         JettyHttpServer httpServer = cluster.getHttp().getHttpServer().orElse(null);
         if (httpServer == null) {
-            httpServer = new JettyHttpServer("DefaultHttpServer", cluster, cluster.isHostedVespa());
+            httpServer = new JettyHttpServer("DefaultHttpServer", cluster, deployState);
             cluster.getHttp().setHttpServer(httpServer);
         }
         int defaultPort = Defaults.getDefaults().vespaWebServicePort();
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index d69fe4fba89..a77a8d1b5b8 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -205,6 +205,7 @@ public class ModelContextImpl implements ModelContext {
         private final int maxCompactBuffers;
         private final boolean failDeploymentWithInvalidJvmOptions;
         private final double tlsSizeFraction;
+        private final List<String> ignoredHttpUserAgents;
         private final boolean enableServerOcspStapling;
 
         public FeatureFlags(FlagSource source, ApplicationId appId) {
@@ -249,6 +250,7 @@ public class ModelContextImpl implements ModelContext {
             this.maxCompactBuffers = flagValue(source, appId, Flags.MAX_COMPACT_BUFFERS);
             this.failDeploymentWithInvalidJvmOptions = flagValue(source, appId, Flags.FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS);
             this.tlsSizeFraction = flagValue(source, appId, Flags.TLS_SIZE_FRACTION);
+            this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS);
             this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING);
         }
 
@@ -295,6 +297,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; }
         @Override public int maxCompactBuffers() { return maxCompactBuffers; }
         @Override public double tlsSizeFraction() { return tlsSizeFraction; }
+        @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; }
         @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) {
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
index b39a3309ad9..75d6cab273a 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
@@ -226,6 +226,12 @@ public class PermanentFlags {
             "Takes effect on next redeployment",
             APPLICATION_ID);
 
+    public static final UnboundListFlag<String> IGNORED_HTTP_USER_AGENTS = defineListFlag(
+            "ignored-http-user-agents", List.of(), String.class,
+            "List of user agents to ignore (crawlers etc)",
+            "Takes effect immediately.",
+            ZONE_ID, APPLICATION_ID);
+
     private PermanentFlags() {}
 
     private static UnboundBooleanFlag defineFeatureFlag(