diff options
10 files changed, 42 insertions, 17 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 9fef9b4615d..a0dba36fef5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -113,6 +113,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default double tlsSizeFraction() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 3ac90673bb6..5c5b065ac29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -94,7 +94,7 @@ public abstract class Container extends AbstractService implements this.index = index; dumpHeapOnShutdownTimeout = deployState.featureFlags().containerDumpHeapOnShutdownTimeout(); shutdownTimeoutS = deployState.featureFlags().containerShutdownTimeout(); - this.defaultHttpServer = new JettyHttpServer("DefaultHttpServer", containerClusterOrNull(parent), deployState.isHosted()); + this.defaultHttpServer = new JettyHttpServer("DefaultHttpServer", containerClusterOrNull(parent), deployState); if (getHttp() == null) { addChild(defaultHttpServer); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index d84884665ef..7b91b4b3244 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.container.bundle.BundleInstantiationSpecification; @@ -25,13 +26,16 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro private final List<ConnectorFactory> connectorFactories = new ArrayList<>(); private final List<String> ignoredUserAgentsList = new ArrayList<>(); - public JettyHttpServer(String componentId, ContainerCluster<?> cluster, boolean isHostedVespa) { + public JettyHttpServer(String componentId, ContainerCluster<?> cluster, DeployState deployState) { super(new ComponentModel(componentId, com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName(), null)); - this.isHostedVespa = isHostedVespa; + this.isHostedVespa = deployState.isHosted(); this.cluster = cluster; final FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(componentId); addChild(filterBindingsProviderComponent); inject(filterBindingsProviderComponent); + for (String agent : deployState.featureFlags().ignoredHttpUserAgents()) { + addIgnoredUserAgent(agent); + } } public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java index 07f4334b542..1518683908b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java @@ -24,7 +24,7 @@ public class JettyHttpServerBuilder extends VespaDomBuilder.DomConfigProducerBui @Override protected JettyHttpServer doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element http) { - JettyHttpServer jettyHttpServer = new JettyHttpServer("jdisc-jetty", cluster, deployState.isHosted()); + JettyHttpServer jettyHttpServer = new JettyHttpServer("jdisc-jetty", cluster, deployState); for (Element serverSpec: XML.getChildren(http, "server")) { ConnectorFactory connectorFactory = new JettyConnectorBuilder().build(deployState, ancestor, serverSpec); jettyHttpServer.addConnector(connectorFactory); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 2da68262fe7..d5c64689648 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -426,7 +426,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.setHttp(buildHttp(deployState, cluster, httpElement)); } if (isHostedTenantApplication(context)) { - addHostedImplicitHttpIfNotPresent(cluster); + addHostedImplicitHttpIfNotPresent(deployState, cluster); addHostedImplicitAccessControlIfNotPresent(deployState, cluster); addDefaultConnectorHostedFilterBinding(cluster); addAdditionalHostedConnector(deployState, cluster, context); @@ -488,13 +488,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return deployState.isHosted() && context.getApplicationType() == ApplicationType.DEFAULT && !isTesterApplication; } - private static void addHostedImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) { + private static void addHostedImplicitHttpIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) { if (cluster.getHttp() == null) { cluster.setHttp(new Http(new FilterChains(cluster))); } JettyHttpServer httpServer = cluster.getHttp().getHttpServer().orElse(null); if (httpServer == null) { - httpServer = new JettyHttpServer("DefaultHttpServer", cluster, cluster.isHostedVespa()); + httpServer = new JettyHttpServer("DefaultHttpServer", cluster, deployState); cluster.getHttp().setHttpServer(httpServer); } int defaultPort = Defaults.getDefaults().vespaWebServicePort(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index d69fe4fba89..a77a8d1b5b8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -205,6 +205,7 @@ public class ModelContextImpl implements ModelContext { private final int maxCompactBuffers; private final boolean failDeploymentWithInvalidJvmOptions; private final double tlsSizeFraction; + private final List<String> ignoredHttpUserAgents; private final boolean enableServerOcspStapling; public FeatureFlags(FlagSource source, ApplicationId appId) { @@ -249,6 +250,7 @@ public class ModelContextImpl implements ModelContext { this.maxCompactBuffers = flagValue(source, appId, Flags.MAX_COMPACT_BUFFERS); this.failDeploymentWithInvalidJvmOptions = flagValue(source, appId, Flags.FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS); this.tlsSizeFraction = flagValue(source, appId, Flags.TLS_SIZE_FRACTION); + this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); } @@ -295,6 +297,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public double tlsSizeFraction() { return tlsSizeFraction; } + @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; } @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index b39a3309ad9..75d6cab273a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -226,6 +226,12 @@ public class PermanentFlags { "Takes effect on next redeployment", APPLICATION_ID); + public static final UnboundListFlag<String> IGNORED_HTTP_USER_AGENTS = defineListFlag( + "ignored-http-user-agents", List.of(), String.class, + "List of user agents to ignore (crawlers etc)", + "Takes effect immediately.", + ZONE_ID, APPLICATION_ID); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/juniper/src/vespa/juniper/sumdesc.cpp b/juniper/src/vespa/juniper/sumdesc.cpp index 1a27ffaefb0..969616423e8 100644 --- a/juniper/src/vespa/juniper/sumdesc.cpp +++ b/juniper/src/vespa/juniper/sumdesc.cpp @@ -18,6 +18,17 @@ LOG_SETUP(".juniper.sumdesc"); namespace { +static constexpr char replacement_char = '.'; + +char printable_char(char c) +{ + unsigned char uc = (unsigned char) c; + if (uc >= 0x80 || uc < (unsigned char) ' ') { + return replacement_char; + } + return c; +} + bool wordchar(const unsigned char* s) { unsigned char c = *s; @@ -98,7 +109,7 @@ int complete_word(unsigned char* start, ssize_t length, // the read: for (;;) { LOG(spam, "[%s%d%s%c]", (whitespace_elim ? "^" : ""), - moved, (increment > 0 ? "+" : "-"), *ptr); + moved, (increment > 0 ? "+" : "-"), printable_char(*ptr)); int cur_move = Fast_UnicodeUtil::UTF8move(start, length, ptr, increment); @@ -114,11 +125,11 @@ int complete_word(unsigned char* start, ssize_t length, // Give up if we found a split of a word if (cur_move <= 0) // == 0 to avoid UTF8move bug in fastlib 1.3.3.. { - LOG(spam, "complete_word: Failing at char %c/0x%x", *ptr, *ptr); + LOG(spam, "complete_word: Failing at char %c/0x%x", printable_char(*ptr), *ptr); break; } if (chartest(ptr)) { - LOG(spam, "complete_word: Breaking at char %c/0x%x (%d)", *ptr, + LOG(spam, "complete_word: Breaking at char %c/0x%x (%d)", printable_char(*ptr), *ptr, cur_move); // count this character (it is the first blank/wordchar) // only if we are going forward and it is a word character @@ -459,12 +470,12 @@ int SummaryDesc::complete_extended_token(unsigned char* start, ssize_t length, // Handle default case ("ordinary" space) if (!word_connector(preptr)) { - LOG(spam, "Not a word connector case (%c)", *preptr); + LOG(spam, "Not a word connector case (%c)", printable_char(*preptr)); return moved; } char wconn = *preptr; (void) wconn; - LOG(spam, "Found word connector case candidate (%c)", wconn); + LOG(spam, "Found word connector case candidate (%c)", printable_char(wconn)); // Read the character before/after the connector character: int addlen = Fast_UnicodeUtil::UTF8move(start, length, @@ -498,7 +509,7 @@ int SummaryDesc::complete_extended_token(unsigned char* start, ssize_t length, ptr = preptr; LOG(spam, "Found proper word connector case (%c,%c) yet moved %d", - wconn, *preptr, moved); + printable_char(wconn), printable_char(*preptr), moved); } } @@ -590,7 +601,7 @@ std::string SummaryDesc::get_summary(const char* buffer, size_t bytes, } else if (!d._highlight) { LOG(spam, "Not completing word at " "char %c/0x%x, prev_end %" PRId64 ", pos %" PRId64, - *ptr, *ptr, static_cast<int64_t>(prev_end), static_cast<int64_t>(pos)); + printable_char(*ptr), *ptr, static_cast<int64_t>(prev_end), static_cast<int64_t>(pos)); } /* Point to "current" endpos to check for split word/ending @@ -616,7 +627,7 @@ std::string SummaryDesc::get_summary(const char* buffer, size_t bytes, } else if (!d._highlight) { LOG(spam, "Not completing word at " "char %c/0x%x, next_pos %" PRId64, - *ptr, *ptr, static_cast<int64_t>(next_pos)); + printable_char(*ptr), *ptr, static_cast<int64_t>(next_pos)); } JD_INVAR(JD_DESC, len >= 0, len = 0, diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.h b/searchlib/src/vespa/searchlib/attribute/changevector.h index 271497398a8..d929e8615a9 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.h +++ b/searchlib/src/vespa/searchlib/attribute/changevector.h @@ -25,7 +25,7 @@ struct ChangeBase { DIV, CLEARDOC }; - enum {UNSET_ENTRY_REF = 0xffffffffu}; + enum {UNSET_ENTRY_REF = 0}; ChangeBase() : _type(NOOP), diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 79957217267..f9301557c0c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -392,7 +392,7 @@ DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, vespalib::stringref input, G std::ostringstream hexDump; hexDump << vespalib::HexDump(input.data(), input.length()); LOG(spam, "makeDynamicTeaser: docid=%d, input='%s', hexdump:\n%s", - docid, input.data(), hexDump.str().c_str()); + docid, std::string(input.data(), input.length()).c_str(), hexDump.str().c_str()); } auto langid = static_cast<uint32_t>(-1); |