Merge pull request #16368 from vespa-engine/bjorncs/hosted-configserver-controller-access-log

Bjorncs/hosted configserver controller access log

10 files changed, 68 insertions, 15 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java
index 880e0e8c574..081f655b369 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java
@@ -15,9 +15,11 @@ import com.yahoo.vespa.model.container.component.AccessLogComponent.CompressionT
  */
 public class LogserverContainer extends Container {
 
-    public LogserverContainer(AbstractConfigProducer parent, FeatureFlags featureFlags, boolean isHostedVespa) {
+    public LogserverContainer(AbstractConfigProducer<?> parent, FeatureFlags featureFlags, boolean isHostedVespa) {
         super(parent, featureFlags, "" + 0, 0, isHostedVespa);
-        addComponent(new AccessLogComponent(AccessLogType.jsonAccessLog, CompressionType.GZIP, ((LogserverContainerCluster) parent).getName(), true));
+        LogserverContainerCluster cluster = (LogserverContainerCluster) parent;
+        addComponent(new AccessLogComponent(
+                cluster, AccessLogType.jsonAccessLog, CompressionType.GZIP, cluster.getName(), true));
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
index 2ad3bc8f84a..4cb3dde0833 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
@@ -55,7 +55,7 @@ public class ClusterControllerContainer extends Container implements
                    "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler",
                    "/cluster/v2/*",
                    CLUSTERCONTROLLER_BUNDLE);
-        addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog,
+        addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog,
                 AccessLogComponent.CompressionType.GZIP,
                 "controller",
                 deployState.isHosted()));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index 8b85c0a46ae..a6d0dad5ff6 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -534,7 +534,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
         var compressionType = isHostedVespa && deployState.featureFlags().enableZstdCompressionAccessLog()
                 ? AccessLogComponent.CompressionType.ZSTD
                 : AccessLogComponent.CompressionType.GZIP;
-        addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, getName(), isHostedVespa));
+        addComponent(new AccessLogComponent(this, AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, getName(), isHostedVespa));
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java
index ffb7b876fa2..2905471b02e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java
@@ -3,9 +3,12 @@ package com.yahoo.vespa.model.container.component;
 
 import com.yahoo.container.core.AccessLogConfig;
 import com.yahoo.container.core.AccessLogConfig.FileHandler.CompressionFormat;
-import com.yahoo.container.logging.VespaAccessLog;
 import com.yahoo.container.logging.JSONAccessLog;
+import com.yahoo.container.logging.VespaAccessLog;
 import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.ContainerCluster;
+
+import java.util.OptionalInt;
 
 /**
  * @author Tony Vaagenes
@@ -13,6 +16,7 @@ import com.yahoo.osgi.provider.model.ComponentModel;
  */
 public final class AccessLogComponent extends SimpleComponent implements AccessLogConfig.Producer {
 
+
     public enum AccessLogType { queryAccessLog, yApacheAccessLog, jsonAccessLog }
     public enum CompressionType { GZIP, ZSTD }
 
@@ -22,10 +26,11 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL
     private final boolean isHostedVespa;
     private final String symlinkName;
     private final CompressionType compressionType;
+    private final int queueSize;
 
-    public AccessLogComponent(AccessLogType logType, CompressionType compressionType, String clusterName, boolean isHostedVespa)
+    public AccessLogComponent(ContainerCluster<?> cluster, AccessLogType logType, CompressionType compressionType, String clusterName, boolean isHostedVespa)
     {
-        this(logType, compressionType,
+        this(cluster, logType, compressionType,
                 String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"),
                 null, null, isHostedVespa,
                 capitalize(logType.name()) + "." + clusterName);
@@ -35,7 +40,8 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL
         return name.substring(0, 1).toUpperCase() + name.substring(1);
     }
 
-    public AccessLogComponent(AccessLogType logType,
+    public AccessLogComponent(ContainerCluster<?> cluster,
+                              AccessLogType logType,
                               CompressionType compressionType,
                               String fileNamePattern,
                               String rotationInterval,
@@ -50,11 +56,19 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL
         this.isHostedVespa = isHostedVespa;
         this.symlinkName = symlinkName;
         this.compressionType = compressionType;
+        this.queueSize = queueSize(cluster).orElse(-1);
 
         if (fileNamePattern == null)
             throw new RuntimeException("File name pattern required when configuring access log.");
     }
 
+    private static OptionalInt queueSize(ContainerCluster<?> cluster) {
+        if (cluster == null) return OptionalInt.empty();
+        double vcpu = cluster.vcpu().orElse(0);
+        if (vcpu <= 0) return OptionalInt.empty();
+        return OptionalInt.of((int) Math.max(4096, Math.ceil(vcpu * 256.0)));
+    }
+
     private static String accessLogClass(AccessLogType logType) {
         switch (logType) {
             case queryAccessLog:
@@ -84,6 +98,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL
         } else if (isHostedVespa) {
             builder.compressOnRotation(true);
         }
+        if (queueSize >= 0) {
+            builder.queueSize(queueSize);
+        }
         switch (compressionType) {
             case GZIP:
                 builder.compressionFormat(CompressionFormat.GZIP);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java
index bf279c5c364..9977e955f85 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java
@@ -5,18 +5,37 @@ package com.yahoo.vespa.model.container.component;
 import com.yahoo.container.logging.ConnectionLog;
 import com.yahoo.container.logging.ConnectionLogConfig;
 import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.ContainerCluster;
+
+import java.util.OptionalInt;
 
 public class ConnectionLogComponent extends SimpleComponent implements ConnectionLogConfig.Producer {
 
     private final String clusterName;
+    private final int queueSize;
+
+    public ConnectionLogComponent(ContainerCluster<?> cluster, Class<? extends ConnectionLog> cls) {
+        this(cluster, cls, cluster.getName());
+    }
 
-    public ConnectionLogComponent(Class<? extends ConnectionLog> cls, String clusterName) {
+    public ConnectionLogComponent(ContainerCluster<?> cluster, Class<? extends ConnectionLog> cls, String clusterName) {
         super(new ComponentModel(cls.getName(), null, "jdisc_http_service", null));
         this.clusterName = clusterName;
+        this.queueSize = queueSize(cluster).orElse(-1);
+    }
+
+    private static OptionalInt queueSize(ContainerCluster<?> cluster) {
+        if (cluster == null) return OptionalInt.empty();
+        double vcpu = cluster.vcpu().orElse(0);
+        if (vcpu <= 0) return OptionalInt.empty();
+        return OptionalInt.of((int) Math.max(4096, Math.ceil(vcpu * 512.0)));
     }
 
     @Override
     public void getConfig(ConnectionLogConfig.Builder builder) {
         builder.cluster(clusterName);
+        if (queueSize >= 0) {
+            builder.queueSize(queueSize);
+        }
     }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
index 901c422f843..b6ebd73b442 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
@@ -25,7 +25,7 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro
     private final ContainerCluster<?> cluster;
     private final boolean isHostedVespa;
     private final List<ConnectorFactory> connectorFactories = new ArrayList<>();
-    private final boolean enableJdiscConnectionLog;
+    private volatile boolean enableJdiscConnectionLog;
 
     public JettyHttpServer(ComponentId id, ContainerCluster<?> cluster, ModelContext.FeatureFlags featureFlags, boolean isHostedVespa) {
         super(new ComponentModel(
@@ -54,6 +54,8 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro
         connectorFactories.remove(connectorFactory);
     }
 
+    public void enableConnectionLog(boolean enabled) { this.enableJdiscConnectionLog = enabled; }
+
     public List<ConnectorFactory> getConnectorFactories() {
         return Collections.unmodifiableList(connectorFactories);
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java
index a66d98ad52c..81e96760684 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java
@@ -55,6 +55,7 @@ public class AccessLogBuilder {
         @Override
         protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element spec) {
             return new AccessLogComponent(
+                    (ContainerCluster<?>) ancestor,
                     accessLogType,
                     compressionType(spec, deployState, isHostedVespa),
                     fileNamePattern(spec),
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
index 8be57ff3bde..0c5375edf71 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
@@ -47,13 +47,22 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder {
         if (isHosted()){
             cluster.addComponent(
                     new AccessLogComponent(
-                            AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD,
+                            cluster, AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD,
                             "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log"));
-            cluster.addComponent(new ConnectionLogComponent(FileConnectionLog.class, cluster.getName()));
+            cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class));
         } else {
             super.addAccessLogs(deployState, cluster, spec);
         }
     }
 
+    @Override
+    protected void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) {
+        super.addHttp(deployState, spec, cluster, context);
+        if (isHosted()) {
+            cluster.getHttp().getHttpServer().get().enableConnectionLog(true);
+        }
+    }
+
+    /** Note: using {@link CloudConfigOptions} as {@link DeployState#isHosted()} returns <em>false</em> for hosted configserver/controller */
     private boolean isHosted() { return options.hostedVespa().orElse(Boolean.FALSE); }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 7477ccdd970..8735b76100d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -350,9 +350,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
 
         // Add connection log if access log is configured
         if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) {
-            cluster.addComponent(new ConnectionLogComponent(FileConnectionLog.class, cluster.getName()));
+            cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class));
         } else {
-            cluster.addComponent(new ConnectionLogComponent(VoidConnectionLog.class, cluster.getName()));
+            cluster.addComponent(new ConnectionLogComponent(cluster, VoidConnectionLog.class));
         }
     }
 
@@ -361,7 +361,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
     }
 
 
-    private void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) {
+    protected void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) {
         Element httpElement = XML.getChild(spec, "http");
         if (httpElement != null) {
             cluster.setHttp(buildHttp(deployState, cluster, httpElement));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
index 1d2dd94d6a3..56f09eefe82 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
@@ -86,6 +86,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase {
             AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler();
             assertEquals("pattern", fileHandlerConfig.pattern());
             assertEquals("interval", fileHandlerConfig.rotation());
+            assertEquals(10000, fileHandlerConfig.queueSize());
         }
 
         { // json
@@ -96,6 +97,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase {
             AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler();
             assertEquals("pattern", fileHandlerConfig.pattern());
             assertEquals("interval", fileHandlerConfig.rotation());
+            assertEquals(10000, fileHandlerConfig.queueSize());
         }
     }
 
@@ -114,6 +116,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase {
         assertNotNull(connectionLogComponent);
         ConnectionLogConfig config = root.getConfig(ConnectionLogConfig.class, "default/component/com.yahoo.container.logging.FileConnectionLog");
         assertEquals("default", config.cluster());
+        assertEquals(10000, config.queueSize());
     }
 
     @Test