From 9bd5a12c87d0a92d37b2c75a9530e96e104b7291 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Wed, 3 Feb 2021 14:17:50 +0100 Subject: Make queue size of log file handler configurable Add new config parameter for connection log and request log. Scale queue size on vcpu in config model. Use LinkedBlockingQueue instead of ArrayBlockingQueue. --- .../vespa/model/admin/LogserverContainer.java | 6 ++++-- .../ClusterControllerContainer.java | 2 +- .../vespa/model/container/ContainerCluster.java | 2 +- .../container/component/AccessLogComponent.java | 25 ++++++++++++++++++---- .../component/ConnectionLogComponent.java | 21 +++++++++++++++++- .../model/container/xml/AccessLogBuilder.java | 1 + .../xml/ConfigServerContainerModelBuilder.java | 4 ++-- .../model/container/xml/ContainerModelBuilder.java | 4 ++-- 8 files changed, 52 insertions(+), 13 deletions(-) (limited to 'config-model/src/main/java/com') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index 880e0e8c574..081f655b369 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -15,9 +15,11 @@ import com.yahoo.vespa.model.container.component.AccessLogComponent.CompressionT */ public class LogserverContainer extends Container { - public LogserverContainer(AbstractConfigProducer parent, FeatureFlags featureFlags, boolean isHostedVespa) { + public LogserverContainer(AbstractConfigProducer parent, FeatureFlags featureFlags, boolean isHostedVespa) { super(parent, featureFlags, "" + 0, 0, isHostedVespa); - addComponent(new AccessLogComponent(AccessLogType.jsonAccessLog, CompressionType.GZIP, ((LogserverContainerCluster) parent).getName(), true)); + LogserverContainerCluster cluster = (LogserverContainerCluster) parent; + addComponent(new AccessLogComponent( + cluster, AccessLogType.jsonAccessLog, CompressionType.GZIP, cluster.getName(), true)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 2ad3bc8f84a..4cb3dde0833 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -55,7 +55,7 @@ public class ClusterControllerContainer extends Container implements "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler", "/cluster/v2/*", CLUSTERCONTROLLER_BUNDLE); - addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, + addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.GZIP, "controller", deployState.isHosted())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 8b85c0a46ae..a6d0dad5ff6 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -534,7 +534,7 @@ public abstract class ContainerCluster var compressionType = isHostedVespa && deployState.featureFlags().enableZstdCompressionAccessLog() ? AccessLogComponent.CompressionType.ZSTD : AccessLogComponent.CompressionType.GZIP; - addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, getName(), isHostedVespa)); + addComponent(new AccessLogComponent(this, AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, getName(), isHostedVespa)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index ffb7b876fa2..7b3d307c0b0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -3,9 +3,12 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.container.core.AccessLogConfig; import com.yahoo.container.core.AccessLogConfig.FileHandler.CompressionFormat; -import com.yahoo.container.logging.VespaAccessLog; import com.yahoo.container.logging.JSONAccessLog; +import com.yahoo.container.logging.VespaAccessLog; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.ContainerCluster; + +import java.util.OptionalInt; /** * @author Tony Vaagenes @@ -13,6 +16,7 @@ import com.yahoo.osgi.provider.model.ComponentModel; */ public final class AccessLogComponent extends SimpleComponent implements AccessLogConfig.Producer { + public enum AccessLogType { queryAccessLog, yApacheAccessLog, jsonAccessLog } public enum CompressionType { GZIP, ZSTD } @@ -22,10 +26,11 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL private final boolean isHostedVespa; private final String symlinkName; private final CompressionType compressionType; + private final int queueSize; - public AccessLogComponent(AccessLogType logType, CompressionType compressionType, String clusterName, boolean isHostedVespa) + public AccessLogComponent(ContainerCluster cluster, AccessLogType logType, CompressionType compressionType, String clusterName, boolean isHostedVespa) { - this(logType, compressionType, + this(cluster, logType, compressionType, String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"), null, null, isHostedVespa, capitalize(logType.name()) + "." + clusterName); @@ -35,7 +40,8 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL return name.substring(0, 1).toUpperCase() + name.substring(1); } - public AccessLogComponent(AccessLogType logType, + public AccessLogComponent(ContainerCluster cluster, + AccessLogType logType, CompressionType compressionType, String fileNamePattern, String rotationInterval, @@ -50,11 +56,19 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; this.compressionType = compressionType; + this.queueSize = queueSize(cluster).orElse(-1); if (fileNamePattern == null) throw new RuntimeException("File name pattern required when configuring access log."); } + private static OptionalInt queueSize(ContainerCluster cluster) { + if (cluster == null) return OptionalInt.empty(); + double vcpu = cluster.vcpu().orElse(0); + if (vcpu <= 0) return OptionalInt.empty(); + return OptionalInt.of((int) Math.max(512, Math.ceil(vcpu * 64.0))); + } + private static String accessLogClass(AccessLogType logType) { switch (logType) { case queryAccessLog: @@ -84,6 +98,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL } else if (isHostedVespa) { builder.compressOnRotation(true); } + if (queueSize >= 0) { + builder.queueSize(queueSize); + } switch (compressionType) { case GZIP: builder.compressionFormat(CompressionFormat.GZIP); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java index bf279c5c364..2441bb77515 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java @@ -5,18 +5,37 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.container.logging.ConnectionLog; import com.yahoo.container.logging.ConnectionLogConfig; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.ContainerCluster; + +import java.util.OptionalInt; public class ConnectionLogComponent extends SimpleComponent implements ConnectionLogConfig.Producer { private final String clusterName; + private final int queueSize; + + public ConnectionLogComponent(ContainerCluster cluster, Class cls) { + this(cluster, cls, cluster.getName()); + } - public ConnectionLogComponent(Class cls, String clusterName) { + public ConnectionLogComponent(ContainerCluster cluster, Class cls, String clusterName) { super(new ComponentModel(cls.getName(), null, "jdisc_http_service", null)); this.clusterName = clusterName; + this.queueSize = queueSize(cluster).orElse(-1); + } + + private static OptionalInt queueSize(ContainerCluster cluster) { + if (cluster == null) return OptionalInt.empty(); + double vcpu = cluster.vcpu().orElse(0); + if (vcpu <= 0) return OptionalInt.empty(); + return OptionalInt.of((int) Math.max(512, Math.ceil(vcpu * 64.0))); } @Override public void getConfig(ConnectionLogConfig.Builder builder) { builder.cluster(clusterName); + if (queueSize >= 0) { + builder.queueSize(queueSize); + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index a66d98ad52c..81e96760684 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -55,6 +55,7 @@ public class AccessLogBuilder { @Override protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element spec) { return new AccessLogComponent( + (ContainerCluster) ancestor, accessLogType, compressionType(spec, deployState, isHostedVespa), fileNamePattern(spec), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 79c0d3e6356..0c5375edf71 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -47,9 +47,9 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { if (isHosted()){ cluster.addComponent( new AccessLogComponent( - AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, + cluster, AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log")); - cluster.addComponent(new ConnectionLogComponent(FileConnectionLog.class, cluster.getName())); + cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class)); } else { super.addAccessLogs(deployState, cluster, spec); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 123bc486b0f..8735b76100d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -350,9 +350,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder { // Add connection log if access log is configured if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) { - cluster.addComponent(new ConnectionLogComponent(FileConnectionLog.class, cluster.getName())); + cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class)); } else { - cluster.addComponent(new ConnectionLogComponent(VoidConnectionLog.class, cluster.getName())); + cluster.addComponent(new ConnectionLogComponent(cluster, VoidConnectionLog.class)); } } -- cgit v1.2.3