From 03087d0a5d7a6ca39b8c21f950b46d0bc757e178 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 28 May 2021 16:02:34 +0200 Subject: Add control of buffer size and queue size of accesslog. Use 1024 and 256k for host-admin. --- .../model/container/component/AccessLogComponent.java | 14 ++++++++------ .../model/container/component/ConnectionLogComponent.java | 1 - .../yahoo/vespa/model/container/xml/AccessLogBuilder.java | 14 +++++++++++++- .../container/xml/ConfigServerContainerModelBuilder.java | 2 +- .../src/main/resources/schema/containercluster.rnc | 3 ++- .../com/yahoo/vespa/model/container/xml/AccessLogTest.java | 10 +++++----- config-model/src/test/schema-test-files/services.xml | 2 +- node-admin/src/main/application/services.xml | 2 +- 8 files changed, 31 insertions(+), 17 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index f18f10644ca..fc41da43479 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -35,7 +35,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this(cluster, logType, compressionType, String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"), null, null, isHostedVespa, - capitalize(logType.name()) + "." + clusterName); + capitalize(logType.name()) + "." + clusterName, + queueSize(cluster).orElse(-1), + ((cluster instanceof ApplicationContainerCluster) ? 4*1024*1024 : null)); } private static String capitalize(String name) { @@ -49,7 +51,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL String rotationInterval, Boolean compressOnRotation, boolean isHostedVespa, - String symlinkName) + String symlinkName, + Integer queueSize, + Integer bufferSize) { super(new ComponentModel(accessLogClass(logType), null, "container-core", null)); this.fileNamePattern = fileNamePattern; @@ -58,10 +62,8 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; this.compressionType = compressionType; - this.queueSize = queueSize(cluster).orElse(-1); - bufferSize = (cluster instanceof ApplicationContainerCluster) - ? 4*1024*1024 - : null; + this.queueSize = (queueSize == null) ? 256 : queueSize; + this.bufferSize = bufferSize; if (fileNamePattern == null) throw new RuntimeException("File name pattern required when configuring access log."); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java index 83e06aab703..0b51cd163a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.container.logging.ConnectionLog; import com.yahoo.container.logging.ConnectionLogConfig; -import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import java.util.OptionalInt; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index f84d6f0724b..d7812e9b4ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -62,7 +62,9 @@ public class AccessLogBuilder { rotationInterval(spec), compressOnRotation(spec), isHostedVespa, - symlinkName(spec)); + symlinkName(spec), + queueSize(spec), + bufferSize(spec)); } private String symlinkName(Element spec) { @@ -74,6 +76,16 @@ public class AccessLogBuilder { return (compress.isEmpty() ? null : Boolean.parseBoolean(compress)); } + private Integer bufferSize(Element spec) { + String value = spec.getAttribute("bufferSize"); + return (value.isEmpty() ? null : Integer.parseInt(value)); + } + + private Integer queueSize(Element spec) { + String value = spec.getAttribute("queueSize"); + return (value.isEmpty() ? null : Integer.parseInt(value)); + } + private String rotationInterval(Element spec) { return nullIfEmpty(spec.getAttribute("rotationInterval")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 7e0be6e448b..a37d0ef416f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -48,7 +48,7 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { cluster.addComponent( new AccessLogComponent( cluster, AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, - "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log")); + "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log", 1024,256*1024)); cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "configserver")); } else { super.addAccessLogs(deployState, cluster, spec); diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 39df939f78c..3679e53e257 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -81,7 +81,8 @@ AccessLog = element accesslog { attribute compressOnRotation { xsd:boolean }? & attribute symlinkName { string }? & attribute compressionType { string "gzip" | string "zstd" }? & - + attribute queueSize { xsd:nonNegativeInteger }? & + attribute bufferSize { xsd:nonNegativeInteger }? & attribute rotationInterval { string }? } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index f3199f6a46f..80aafdd4ec7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -69,7 +69,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { " ", " ", + " fileNamePattern='pattern' rotationInterval='interval' queueSize='17' bufferSize='65536'/>", nodesXml, "" ); @@ -85,8 +85,8 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler(); assertEquals("pattern", fileHandlerConfig.pattern()); assertEquals("interval", fileHandlerConfig.rotation()); - assertEquals(10000, fileHandlerConfig.queueSize()); - assertEquals(4*1024*1024, fileHandlerConfig.bufferSize()); + assertEquals(256, fileHandlerConfig.queueSize()); + assertEquals(256*1024, fileHandlerConfig.bufferSize()); } { // json @@ -97,8 +97,8 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler(); assertEquals("pattern", fileHandlerConfig.pattern()); assertEquals("interval", fileHandlerConfig.rotation()); - assertEquals(10000, fileHandlerConfig.queueSize()); - assertEquals(4*1024*1024, fileHandlerConfig.bufferSize()); + assertEquals(17, fileHandlerConfig.queueSize()); + assertEquals(65536, fileHandlerConfig.bufferSize()); } } diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index db1e6c29586..4ad7865c93b 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -147,7 +147,7 @@ + fileNamePattern='logs/vespa/qrs/access-json.%Y%m%d%H%M%S' compressOnRotation='true' compressionType='zstd' queueSize='13' bufferSize='65536'/> diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index 7e1d45c3ff2..d978b358032 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -3,7 +3,7 @@ - + -- cgit v1.2.3