Merge pull request #20512 from vespa-engine/arnej/ignore-some-user-agents

minimal implementation of "ignore some user agents"

5 files changed, 39 insertions, 7 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
index 8c31eabfde0..d84884665ef 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
@@ -23,6 +23,7 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro
     private final ContainerCluster<?> cluster;
     private volatile boolean isHostedVespa;
     private final List<ConnectorFactory> connectorFactories = new ArrayList<>();
+    private final List<String> ignoredUserAgentsList = new ArrayList<>();
 
     public JettyHttpServer(String componentId, ContainerCluster<?> cluster, boolean isHostedVespa) {
         super(new ComponentModel(componentId, com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName(), null));
@@ -44,10 +45,15 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro
         return Collections.unmodifiableList(connectorFactories);
     }
 
+    public void addIgnoredUserAgent(String userAgent) {
+        ignoredUserAgentsList.add(userAgent);
+    }
+
     @Override
     public void getConfig(ServerConfig.Builder builder) {
         builder.metric(new ServerConfig.Metric.Builder()
                 .monitoringHandlerPaths(List.of("/state/v1", "/status.html", "/metrics/v2"))
+                .ignoredUserAgents(ignoredUserAgentsList)
                 .searchHandlerPaths(List.of("/search"))
         );
         if (isHostedVespa) {
diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json
index a6783d1e5f5..c3c946ab537 100644
--- a/container-core/abi-spec.json
+++ b/container-core/abi-spec.json
@@ -1990,11 +1990,14 @@
       "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder monitoringHandlerPaths(java.util.Collection)",
       "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder searchHandlerPaths(java.lang.String)",
       "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder searchHandlerPaths(java.util.Collection)",
+      "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder ignoredUserAgents(java.lang.String)",
+      "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder ignoredUserAgents(java.util.Collection)",
       "public com.yahoo.jdisc.http.ServerConfig$Metric build()"
     ],
     "fields": [
       "public java.util.List monitoringHandlerPaths",
-      "public java.util.List searchHandlerPaths"
+      "public java.util.List searchHandlerPaths",
+      "public java.util.List ignoredUserAgents"
     ]
   },
   "com.yahoo.jdisc.http.ServerConfig$Metric": {
@@ -2009,7 +2012,9 @@
       "public java.util.List monitoringHandlerPaths()",
       "public java.lang.String monitoringHandlerPaths(int)",
       "public java.util.List searchHandlerPaths()",
-      "public java.lang.String searchHandlerPaths(int)"
+      "public java.lang.String searchHandlerPaths(int)",
+      "public java.util.List ignoredUserAgents()",
+      "public java.lang.String ignoredUserAgents(int)"
     ],
     "fields": []
   },
diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java
index 0aa2820f959..631f4080c7e 100644
--- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java
+++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java
@@ -3,6 +3,7 @@ package com.yahoo.jdisc.http.server.jetty;
 
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.http.HttpRequest;
+import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.server.AsyncContextEvent;
 import org.eclipse.jetty.server.Handler;
@@ -21,6 +22,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -106,12 +108,25 @@ class HttpResponseStatisticsCollector extends HandlerWrapper implements Graceful
         }
     }
 
-    private void observeEndOfRequest(Request request, HttpServletResponse flushableResponse) throws IOException {
-        var metrics = StatusCodeMetric.of(request, monitoringHandlerPaths, searchHandlerPaths);
-        metrics.forEach(metric ->
-                statistics.computeIfAbsent(metric, __ -> new LongAdder())
-                        .increment());
+    void ignoreUserAgent(String agentName) {
+        ignoredUserAgents.add(agentName);
+    }
 
+    private Set<String> ignoredUserAgents = new HashSet<>();
+
+    private boolean shouldLogMetricsFor(Request request) {
+        String agent = request.getHeader(HttpHeader.USER_AGENT.toString());
+        if (agent == null) return true;
+        return ! ignoredUserAgents.contains(agent);
+    }
+
+    private void observeEndOfRequest(Request request, HttpServletResponse flushableResponse) throws IOException {
+        if (shouldLogMetricsFor(request)) {
+            var metrics = StatusCodeMetric.of(request, monitoringHandlerPaths, searchHandlerPaths);
+            metrics.forEach(metric ->
+                            statistics.computeIfAbsent(metric, __ -> new LongAdder())
+                            .increment());
+        }
         long live = inFlight.decrementAndGet();
         FutureCallback shutdownCb = shutdown.get();
         if (shutdownCb != null) {
diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java
index fca34f3bbd7..006d6cc84da 100644
--- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java
+++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java
@@ -159,6 +159,9 @@ public class JettyHttpServer extends AbstractServerProvider {
                 new HttpResponseStatisticsCollector(serverConfig.metric().monitoringHandlerPaths(),
                                                     serverConfig.metric().searchHandlerPaths());
         statisticsCollector.setHandler(gzipHandler);
+        for (String agent : serverConfig.metric().ignoredUserAgents()) {
+            statisticsCollector.ignoreUserAgent(agent);
+        }
 
         StatisticsHandler statisticsHandler = newStatisticsHandler();
         statisticsHandler.setHandler(statisticsCollector);
diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def
index db7322c76f2..7e1404bbc5e 100644
--- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def
+++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def
@@ -57,6 +57,9 @@ metric.monitoringHandlerPaths[]       string
 # Paths that should be reported with search dimensions where applicable
 metric.searchHandlerPaths[]           string
 
+# User-agent names to ignore wrt statistics (crawlers etc)
+metric.ignoredUserAgents[]            string
+
 # HTTP request headers that contain remote address
 accessLog.remoteAddressHeaders[]      string