Make queue size of log file handler configurable

Add new config parameter for connection log and request log.
Scale queue size on vcpu in config model.
Use LinkedBlockingQueue instead of ArrayBlockingQueue.

9 files changed, 55 insertions, 13 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java
index 880e0e8c574..081f655b369 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java
@@ -15,9 +15,11 @@ import com.yahoo.vespa.model.container.component.AccessLogComponent.CompressionT
  */
 public class LogserverContainer extends Container {
 
-    public LogserverContainer(AbstractConfigProducer parent, FeatureFlags featureFlags, boolean isHostedVespa) {
+    public LogserverContainer(AbstractConfigProducer<?> parent, FeatureFlags featureFlags, boolean isHostedVespa) {
         super(parent, featureFlags, "" + 0, 0, isHostedVespa);
-        addComponent(new AccessLogComponent(AccessLogType.jsonAccessLog, CompressionType.GZIP, ((LogserverContainerCluster) parent).getName(), true));
+        LogserverContainerCluster cluster = (LogserverContainerCluster) parent;
+        addComponent(new AccessLogComponent(
+                cluster, AccessLogType.jsonAccessLog, CompressionType.GZIP, cluster.getName(), true));
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
index 2ad3bc8f84a..4cb3dde0833 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
@@ -55,7 +55,7 @@ public class ClusterControllerContainer extends Container implements
                    "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler",
                    "/cluster/v2/*",
                    CLUSTERCONTROLLER_BUNDLE);
-        addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog,
+        addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog,
                 AccessLogComponent.CompressionType.GZIP,
                 "controller",
                 deployState.isHosted()));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index 8b85c0a46ae..a6d0dad5ff6 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -534,7 +534,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
         var compressionType = isHostedVespa && deployState.featureFlags().enableZstdCompressionAccessLog()
                 ? AccessLogComponent.CompressionType.ZSTD
                 : AccessLogComponent.CompressionType.GZIP;
-        addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, getName(), isHostedVespa));
+        addComponent(new AccessLogComponent(this, AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, getName(), isHostedVespa));
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java
index ffb7b876fa2..7b3d307c0b0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java
@@ -3,9 +3,12 @@ package com.yahoo.vespa.model.container.component;
 
 import com.yahoo.container.core.AccessLogConfig;
 import com.yahoo.container.core.AccessLogConfig.FileHandler.CompressionFormat;
-import com.yahoo.container.logging.VespaAccessLog;
 import com.yahoo.container.logging.JSONAccessLog;
+import com.yahoo.container.logging.VespaAccessLog;
 import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.ContainerCluster;
+
+import java.util.OptionalInt;
 
 /**
  * @author Tony Vaagenes
@@ -13,6 +16,7 @@ import com.yahoo.osgi.provider.model.ComponentModel;
  */
 public final class AccessLogComponent extends SimpleComponent implements AccessLogConfig.Producer {
 
+
     public enum AccessLogType { queryAccessLog, yApacheAccessLog, jsonAccessLog }
     public enum CompressionType { GZIP, ZSTD }
 
@@ -22,10 +26,11 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL
     private final boolean isHostedVespa;
     private final String symlinkName;
     private final CompressionType compressionType;
+    private final int queueSize;
 
-    public AccessLogComponent(AccessLogType logType, CompressionType compressionType, String clusterName, boolean isHostedVespa)
+    public AccessLogComponent(ContainerCluster<?> cluster, AccessLogType logType, CompressionType compressionType, String clusterName, boolean isHostedVespa)
     {
-        this(logType, compressionType,
+        this(cluster, logType, compressionType,
                 String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"),
                 null, null, isHostedVespa,
                 capitalize(logType.name()) + "." + clusterName);
@@ -35,7 +40,8 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL
         return name.substring(0, 1).toUpperCase() + name.substring(1);
     }
 
-    public AccessLogComponent(AccessLogType logType,
+    public AccessLogComponent(ContainerCluster<?> cluster,
+                              AccessLogType logType,
                               CompressionType compressionType,
                               String fileNamePattern,
                               String rotationInterval,
@@ -50,11 +56,19 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL
         this.isHostedVespa = isHostedVespa;
         this.symlinkName = symlinkName;
         this.compressionType = compressionType;
+        this.queueSize = queueSize(cluster).orElse(-1);
 
         if (fileNamePattern == null)
             throw new RuntimeException("File name pattern required when configuring access log.");
     }
 
+    private static OptionalInt queueSize(ContainerCluster<?> cluster) {
+        if (cluster == null) return OptionalInt.empty();
+        double vcpu = cluster.vcpu().orElse(0);
+        if (vcpu <= 0) return OptionalInt.empty();
+        return OptionalInt.of((int) Math.max(512, Math.ceil(vcpu * 64.0)));
+    }
+
     private static String accessLogClass(AccessLogType logType) {
         switch (logType) {
             case queryAccessLog:
@@ -84,6 +98,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL
         } else if (isHostedVespa) {
             builder.compressOnRotation(true);
         }
+        if (queueSize >= 0) {
+            builder.queueSize(queueSize);
+        }
         switch (compressionType) {
             case GZIP:
                 builder.compressionFormat(CompressionFormat.GZIP);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java
index bf279c5c364..2441bb77515 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java
@@ -5,18 +5,37 @@ package com.yahoo.vespa.model.container.component;
 import com.yahoo.container.logging.ConnectionLog;
 import com.yahoo.container.logging.ConnectionLogConfig;
 import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.ContainerCluster;
+
+import java.util.OptionalInt;
 
 public class ConnectionLogComponent extends SimpleComponent implements ConnectionLogConfig.Producer {
 
     private final String clusterName;
+    private final int queueSize;
+
+    public ConnectionLogComponent(ContainerCluster<?> cluster, Class<? extends ConnectionLog> cls) {
+        this(cluster, cls, cluster.getName());
+    }
 
-    public ConnectionLogComponent(Class<? extends ConnectionLog> cls, String clusterName) {
+    public ConnectionLogComponent(ContainerCluster<?> cluster, Class<? extends ConnectionLog> cls, String clusterName) {
         super(new ComponentModel(cls.getName(), null, "jdisc_http_service", null));
         this.clusterName = clusterName;
+        this.queueSize = queueSize(cluster).orElse(-1);
+    }
+
+    private static OptionalInt queueSize(ContainerCluster<?> cluster) {
+        if (cluster == null) return OptionalInt.empty();
+        double vcpu = cluster.vcpu().orElse(0);
+        if (vcpu <= 0) return OptionalInt.empty();
+        return OptionalInt.of((int) Math.max(512, Math.ceil(vcpu * 64.0)));
     }
 
     @Override
     public void getConfig(ConnectionLogConfig.Builder builder) {
         builder.cluster(clusterName);
+        if (queueSize >= 0) {
+            builder.queueSize(queueSize);
+        }
     }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java
index a66d98ad52c..81e96760684 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java
@@ -55,6 +55,7 @@ public class AccessLogBuilder {
         @Override
         protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element spec) {
             return new AccessLogComponent(
+                    (ContainerCluster<?>) ancestor,
                     accessLogType,
                     compressionType(spec, deployState, isHostedVespa),
                     fileNamePattern(spec),
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
index 79c0d3e6356..0c5375edf71 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
@@ -47,9 +47,9 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder {
         if (isHosted()){
             cluster.addComponent(
                     new AccessLogComponent(
-                            AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD,
+                            cluster, AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD,
                             "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log"));
-            cluster.addComponent(new ConnectionLogComponent(FileConnectionLog.class, cluster.getName()));
+            cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class));
         } else {
             super.addAccessLogs(deployState, cluster, spec);
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 123bc486b0f..8735b76100d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -350,9 +350,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
 
         // Add connection log if access log is configured
         if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) {
-            cluster.addComponent(new ConnectionLogComponent(FileConnectionLog.class, cluster.getName()));
+            cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class));
         } else {
-            cluster.addComponent(new ConnectionLogComponent(VoidConnectionLog.class, cluster.getName()));
+            cluster.addComponent(new ConnectionLogComponent(cluster, VoidConnectionLog.class));
         }
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
index 1d2dd94d6a3..b11aa00f7c6 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
@@ -86,6 +86,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase {
             AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler();
             assertEquals("pattern", fileHandlerConfig.pattern());
             assertEquals("interval", fileHandlerConfig.rotation());
+            assertEquals(2048, fileHandlerConfig.queueSize());
         }
 
         { // json
@@ -96,6 +97,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase {
             AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler();
             assertEquals("pattern", fileHandlerConfig.pattern());
             assertEquals("interval", fileHandlerConfig.rotation());
+            assertEquals(2048, fileHandlerConfig.queueSize());
         }
     }
 
@@ -114,6 +116,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase {
         assertNotNull(connectionLogComponent);
         ConnectionLogConfig config = root.getConfig(ConnectionLogConfig.class, "default/component/com.yahoo.container.logging.FileConnectionLog");
         assertEquals("default", config.cluster());
+        assertEquals(2048, config.queueSize());
     }
 
     @Test