diff options
7 files changed, 50 insertions, 21 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index a4b01259a72..880e0e8c574 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -6,6 +6,8 @@ import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.component.AccessLogComponent; +import com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType; +import com.yahoo.vespa.model.container.component.AccessLogComponent.CompressionType; /** * Container that should be running on same host as the logserver. Sets up a handler for getting logs from logserver. @@ -15,7 +17,7 @@ public class LogserverContainer extends Container { public LogserverContainer(AbstractConfigProducer parent, FeatureFlags featureFlags, boolean isHostedVespa) { super(parent, featureFlags, "" + 0, 0, isHostedVespa); - addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, ((LogserverContainerCluster) parent).getName(), true, false)); + addComponent(new AccessLogComponent(AccessLogType.jsonAccessLog, CompressionType.GZIP, ((LogserverContainerCluster) parent).getName(), true)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index e2df04ebbf3..2ad3bc8f84a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -56,9 +56,9 @@ public class ClusterControllerContainer extends Container implements "/cluster/v2/*", CLUSTERCONTROLLER_BUNDLE); addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, - "controller", - deployState.isHosted(), - deployState.featureFlags().enableZstdCompressionAccessLog())); + AccessLogComponent.CompressionType.GZIP, + "controller", + deployState.isHosted())); // TODO: Why are bundles added here instead of in the cluster? addFileBundle("clustercontroller-apps"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 23a860e2b67..8b85c0a46ae 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -531,7 +531,10 @@ public abstract class ContainerCluster<CONTAINER extends Container> } public void addDefaultSearchAccessLog(DeployState deployState) { - addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, getName(), isHostedVespa, deployState.featureFlags().enableZstdCompressionAccessLog())); + var compressionType = isHostedVespa && deployState.featureFlags().enableZstdCompressionAccessLog() + ? AccessLogComponent.CompressionType.ZSTD + : AccessLogComponent.CompressionType.GZIP; + addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, getName(), isHostedVespa)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index 6e197915a64..ffb7b876fa2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -14,21 +14,21 @@ import com.yahoo.osgi.provider.model.ComponentModel; public final class AccessLogComponent extends SimpleComponent implements AccessLogConfig.Producer { public enum AccessLogType { queryAccessLog, yApacheAccessLog, jsonAccessLog } + public enum CompressionType { GZIP, ZSTD } private final String fileNamePattern; private final String rotationInterval; private final Boolean compression; private final boolean isHostedVespa; private final String symlinkName; - private final boolean enableZstdCompression; + private final CompressionType compressionType; - public AccessLogComponent(AccessLogType logType, String clusterName, boolean isHostedVespa, boolean enableZstdCompression) + public AccessLogComponent(AccessLogType logType, CompressionType compressionType, String clusterName, boolean isHostedVespa) { - this(logType, + this(logType, compressionType, String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"), null, null, isHostedVespa, - capitalize(logType.name()) + "." + clusterName, - enableZstdCompression); + capitalize(logType.name()) + "." + clusterName); } private static String capitalize(String name) { @@ -36,12 +36,12 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL } public AccessLogComponent(AccessLogType logType, + CompressionType compressionType, String fileNamePattern, String rotationInterval, Boolean compressOnRotation, boolean isHostedVespa, - String symlinkName, - boolean enableZstdCompression) + String symlinkName) { super(new ComponentModel(accessLogClass(logType), null, "container-core", null)); this.fileNamePattern = fileNamePattern; @@ -49,7 +49,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.compression = compressOnRotation; this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; - this.enableZstdCompression = enableZstdCompression; + this.compressionType = compressionType; if (fileNamePattern == null) throw new RuntimeException("File name pattern required when configuring access log."); @@ -84,10 +84,16 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL } else if (isHostedVespa) { builder.compressOnRotation(true); } - if (enableZstdCompression) { - builder.compressionFormat(CompressionFormat.Enum.ZSTD); + switch (compressionType) { + case GZIP: + builder.compressionFormat(CompressionFormat.GZIP); + break; + case ZSTD: + builder.compressionFormat(CompressionFormat.ZSTD); + break; + default: + throw new IllegalArgumentException("Unknown compression type: " + compressionType); } - return builder; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index 5b87dc57306..6a858bd2e02 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType; +import com.yahoo.vespa.model.container.component.AccessLogComponent.CompressionType; import org.w3c.dom.Element; import java.util.Optional; @@ -52,15 +53,15 @@ public class AccessLogBuilder { } @Override - protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element spec) { + protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element spec) { return new AccessLogComponent( accessLogType, + compressionType(spec, deployState, isHostedVespa), fileNamePattern(spec), rotationInterval(spec), compressOnRotation(spec), isHostedVespa, - symlinkName(spec), - deployState.featureFlags().enableZstdCompressionAccessLog()); + symlinkName(spec)); } private String symlinkName(Element spec) { @@ -79,6 +80,22 @@ public class AccessLogBuilder { private String fileNamePattern(Element spec) { return nullIfEmpty(spec.getAttribute("fileNamePattern")); } + + private static CompressionType compressionType(Element spec, DeployState deployState, boolean isHostedVespa) { + return Optional.ofNullable(spec.getAttribute("compressionType")) + .filter(value -> !value.isBlank()) + .map(value -> { + switch (value) { + case "gzip": + return CompressionType.GZIP; + case "zstd": + return CompressionType.ZSTD; + default: + throw new IllegalArgumentException("Unknown compression type: " + value); + } + }) + .orElse(isHostedVespa && deployState.featureFlags().enableZstdCompressionAccessLog() ? CompressionType.ZSTD : CompressionType.GZIP); + } } private static AccessLogType logTypeFor(AccessLogTypeLiteral typeLiteral) { @@ -94,7 +111,7 @@ public class AccessLogBuilder { } } - public static Optional<AccessLogComponent> buildIfNotDisabled(DeployState deployState, ContainerCluster cluster, Element accessLogSpec) { + public static Optional<AccessLogComponent> buildIfNotDisabled(DeployState deployState, ContainerCluster<?> cluster, Element accessLogSpec) { AccessLogTypeLiteral typeLiteral = getOptionalAttribute(accessLogSpec, "type"). map(AccessLogTypeLiteral::fromAttributeValue). diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index ca7efd5a938..5257345c39e 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -80,6 +80,7 @@ AccessLog = element accesslog { attribute fileNamePattern { string }? & attribute compressOnRotation { xsd:boolean }? & attribute symlinkName { string }? & + attribute compressionType { string "gzip" | string "zstd" }? & attribute rotationInterval { string }? } diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 7ae4bba00ce..d37000b1ff7 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -147,7 +147,7 @@ </http> <accesslog type='json' - fileNamePattern='logs/vespa/qrs/access-json.%Y%m%d%H%M%S' /> + fileNamePattern='logs/vespa/qrs/access-json.%Y%m%d%H%M%S' compressOnRotation='true' compressionType='zstd'/> <accesslog type='vespa' fileNamePattern='logs/vespa/qrs/access-vespa.%Y%m%d%H%M%S' /> |