From 0002861d9d2dc033cb454c6cb79720c531d046ce Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Mon, 3 Jan 2022 09:27:36 +0000 Subject: add feature flag for ignored HTTP user agents --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 1 + .../java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java | 3 +++ flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 6c70af8cbca..77bf2ea759a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -113,6 +113,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default double tlsSizeFraction() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List ignoredHttpUserAgents() { return List.of(); } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 063603fe8a8..942a84a2730 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -205,6 +205,7 @@ public class ModelContextImpl implements ModelContext { private final int maxCompactBuffers; private final boolean failDeploymentWithInvalidJvmOptions; private final double tlsSizeFraction; + private final List ignoredHttpUserAgents; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -248,6 +249,7 @@ public class ModelContextImpl implements ModelContext { this.maxCompactBuffers = flagValue(source, appId, Flags.MAX_COMPACT_BUFFERS); this.failDeploymentWithInvalidJvmOptions = flagValue(source, appId, Flags.FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS); this.tlsSizeFraction = flagValue(source, appId, Flags.TLS_SIZE_FRACTION); + this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -293,6 +295,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public double tlsSizeFraction() { return tlsSizeFraction; } + @Override public List ignoredHttpUserAgents() { return ignoredHttpUserAgents; } private static V flagValue(FlagSource source, ApplicationId appId, UnboundFlag flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index b39a3309ad9..75d6cab273a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -226,6 +226,12 @@ public class PermanentFlags { "Takes effect on next redeployment", APPLICATION_ID); + public static final UnboundListFlag IGNORED_HTTP_USER_AGENTS = defineListFlag( + "ignored-http-user-agents", List.of(), String.class, + "List of user agents to ignore (crawlers etc)", + "Takes effect immediately.", + ZONE_ID, APPLICATION_ID); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( -- cgit v1.2.3 From 2ff775ff41d2d759c62f7423a34c795c3ea2a504 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Mon, 3 Jan 2022 09:29:33 +0000 Subject: wire ignored HTTP user agents --- .../src/main/java/com/yahoo/vespa/model/container/Container.java | 2 +- .../com/yahoo/vespa/model/container/http/JettyHttpServer.java | 8 ++++++-- .../vespa/model/container/http/xml/JettyHttpServerBuilder.java | 2 +- .../yahoo/vespa/model/container/xml/ContainerModelBuilder.java | 6 +++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 3ac90673bb6..5c5b065ac29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -94,7 +94,7 @@ public abstract class Container extends AbstractService implements this.index = index; dumpHeapOnShutdownTimeout = deployState.featureFlags().containerDumpHeapOnShutdownTimeout(); shutdownTimeoutS = deployState.featureFlags().containerShutdownTimeout(); - this.defaultHttpServer = new JettyHttpServer("DefaultHttpServer", containerClusterOrNull(parent), deployState.isHosted()); + this.defaultHttpServer = new JettyHttpServer("DefaultHttpServer", containerClusterOrNull(parent), deployState); if (getHttp() == null) { addChild(defaultHttpServer); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index d84884665ef..7b91b4b3244 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.container.bundle.BundleInstantiationSpecification; @@ -25,13 +26,16 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro private final List connectorFactories = new ArrayList<>(); private final List ignoredUserAgentsList = new ArrayList<>(); - public JettyHttpServer(String componentId, ContainerCluster cluster, boolean isHostedVespa) { + public JettyHttpServer(String componentId, ContainerCluster cluster, DeployState deployState) { super(new ComponentModel(componentId, com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName(), null)); - this.isHostedVespa = isHostedVespa; + this.isHostedVespa = deployState.isHosted(); this.cluster = cluster; final FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(componentId); addChild(filterBindingsProviderComponent); inject(filterBindingsProviderComponent); + for (String agent : deployState.featureFlags().ignoredHttpUserAgents()) { + addIgnoredUserAgent(agent); + } } public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java index 07f4334b542..1518683908b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java @@ -24,7 +24,7 @@ public class JettyHttpServerBuilder extends VespaDomBuilder.DomConfigProducerBui @Override protected JettyHttpServer doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element http) { - JettyHttpServer jettyHttpServer = new JettyHttpServer("jdisc-jetty", cluster, deployState.isHosted()); + JettyHttpServer jettyHttpServer = new JettyHttpServer("jdisc-jetty", cluster, deployState); for (Element serverSpec: XML.getChildren(http, "server")) { ConnectorFactory connectorFactory = new JettyConnectorBuilder().build(deployState, ancestor, serverSpec); jettyHttpServer.addConnector(connectorFactory); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 2da68262fe7..d5c64689648 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -426,7 +426,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { cluster.setHttp(buildHttp(deployState, cluster, httpElement)); } if (isHostedTenantApplication(context)) { - addHostedImplicitHttpIfNotPresent(cluster); + addHostedImplicitHttpIfNotPresent(deployState, cluster); addHostedImplicitAccessControlIfNotPresent(deployState, cluster); addDefaultConnectorHostedFilterBinding(cluster); addAdditionalHostedConnector(deployState, cluster, context); @@ -488,13 +488,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder { return deployState.isHosted() && context.getApplicationType() == ApplicationType.DEFAULT && !isTesterApplication; } - private static void addHostedImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) { + private static void addHostedImplicitHttpIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) { if (cluster.getHttp() == null) { cluster.setHttp(new Http(new FilterChains(cluster))); } JettyHttpServer httpServer = cluster.getHttp().getHttpServer().orElse(null); if (httpServer == null) { - httpServer = new JettyHttpServer("DefaultHttpServer", cluster, cluster.isHostedVespa()); + httpServer = new JettyHttpServer("DefaultHttpServer", cluster, deployState); cluster.getHttp().setHttpServer(httpServer); } int defaultPort = Defaults.getDefaults().vespaWebServicePort(); -- cgit v1.2.3