diff options
author | Arne H Juul <arnej@yahooinc.com> | 2022-06-14 11:51:13 +0000 |
---|---|---|
committer | Arne H Juul <arnej@yahooinc.com> | 2022-06-14 11:52:14 +0000 |
commit | e78543c0bdc39dd40f1db348ccf34812a0b20133 (patch) | |
tree | e56283af371da30f914745995367ef842cc7a339 /config-model/src | |
parent | c5558250003dbc6bbfcce36b259b369471594b26 (diff) |
use new feature flag for compression format
* note: all defaults should remain the same in this commit
Diffstat (limited to 'config-model/src')
8 files changed, 27 insertions, 25 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java index c87eeab23a5..d5a0b97a416 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.PortAllocBridge; @@ -16,6 +17,7 @@ public class Logserver extends AbstractService { private static final long serialVersionUID = 1L; private static final String logArchiveDir = "$ROOT/logs/vespa/logarchive"; + private String compressionType = "gzip"; public Logserver(AbstractConfigProducer parent) { super(parent, "logserver"); @@ -27,6 +29,12 @@ public class Logserver extends AbstractService { setProp("clustername", "admin"); } + @Override + public void initService(DeployState deployState) { + super.initService(deployState); + this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm("gzip"); + } + /** * @return the startup command for the logserver */ @@ -44,6 +52,8 @@ public class Logserver extends AbstractService { sb.append("-Dlogserver.rpcListenPort=").append(getRelativePort(0)); sb.append(" "); sb.append("-Dlogserver.logarchive.dir=" + logArchiveDir); + sb.append(" "); + sb.append("-Dlogserver.logarchive.compression=" + compressionType); return sb.toString(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index 251ef48c9f7..e4004cc221f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -23,8 +23,9 @@ public class LogserverContainer extends Container { super(parent, "" + 0, 0, deployState); if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) useDynamicPorts(); LogserverContainerCluster cluster = (LogserverContainerCluster) parent; - addComponent(new AccessLogComponent( - cluster, AccessLogType.jsonAccessLog, CompressionType.GZIP, Optional.of(cluster.getName()), true)); + addComponent(new AccessLogComponent(cluster, AccessLogType.jsonAccessLog, + deployState.featureFlags().logFileCompressionAlgorithm("gzip"), + Optional.of(cluster.getName()), true)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 45e9b3a905a..48d11a078e0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -66,7 +66,7 @@ public class ClusterControllerContainer extends Container implements "/cluster/v2/*", CLUSTERCONTROLLER_BUNDLE); addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, - AccessLogComponent.CompressionType.GZIP, + deployState.featureFlags().logFileCompressionAlgorithm("gzip"), Optional.of("controller"), deployState.isHosted())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index f159362d750..03bdf4eb12a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -59,7 +59,7 @@ public class MetricsProxyContainer extends Container implements setProp("index", String.valueOf(index)); addNodeSpecificComponents(); addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, - AccessLogComponent.CompressionType.ZSTD, + "zstd", Optional.of("metrics-proxy"), deployState.isHosted())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 79412368eae..405cda4494c 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -146,6 +146,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> private final boolean isHostedVespa; private final boolean zooKeeperLocalhostAffinity; private final int numAvailableProcessors; + private final String compressionType; private final Map<String, String> concreteDocumentTypes = new LinkedHashMap<>(); @@ -166,7 +167,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); this.zooKeeperLocalhostAffinity = zooKeeperLocalhostAffinity; - numAvailableProcessors = deployState.featureFlags().availableProcessors(); + this.numAvailableProcessors = deployState.featureFlags().availableProcessors(); + this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm(isHostedVespa ? "zstd" : "gzip"); componentGroup = new ComponentGroup<>(this, "component"); @@ -526,7 +528,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> } public void addDefaultSearchAccessLog() { - var compressionType = isHostedVespa ? AccessLogComponent.CompressionType.ZSTD : AccessLogComponent.CompressionType.GZIP; // In hosted Vespa with one application container per node we do not use the container name to distinguish log files Optional<String> clusterName = isHostedVespa ? Optional.empty() : Optional.of(getName()); addComponent(new AccessLogComponent(this, AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, clusterName, isHostedVespa)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index cc0415a7630..79c108cd867 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -32,7 +32,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL private final int queueSize; private final Integer bufferSize; - public AccessLogComponent(ContainerCluster<?> cluster, AccessLogType logType, CompressionType compressionType, Optional<String> clusterName, boolean isHostedVespa) + public AccessLogComponent(ContainerCluster<?> cluster, AccessLogType logType, String compressionType, Optional<String> clusterName, boolean isHostedVespa) { // In hosted Vespa we do not use the clusterName when setting up application ContainerCluster logging this(logType, @@ -55,7 +55,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL } public AccessLogComponent(AccessLogType logType, - CompressionType compressionType, + String compressionType, String fileNamePattern, String rotationInterval, Boolean compressOnRotation, @@ -70,7 +70,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.compression = compressOnRotation; this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; - this.compressionType = compressionType; + this.compressionType = "zstd".equals(compressionType) ? CompressionType.ZSTD :CompressionType.GZIP; this.queueSize = (queueSize == null) ? 256 : queueSize; this.bufferSize = bufferSize; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index b939109dab1..bd2627f29d0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -54,9 +54,10 @@ public class AccessLogBuilder { @Override protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element spec) { + String fallback = deployState.featureFlags().logFileCompressionAlgorithm(isHostedVespa ? "zstd" : "gzip"); return new AccessLogComponent( accessLogType, - compressionType(spec, isHostedVespa), + compressionType(spec, fallback), fileNamePattern(spec), rotationInterval(spec), compressOnRotation(spec), @@ -93,21 +94,10 @@ public class AccessLogBuilder { return nullIfEmpty(spec.getAttribute("fileNamePattern")); } - private static CompressionType compressionType(Element spec, boolean isHostedVespa) { - CompressionType fallback = isHostedVespa ? CompressionType.ZSTD : CompressionType.GZIP; + private static String compressionType(Element spec, String fallback) { return Optional.ofNullable(spec.getAttribute("compressionType")) - .filter(value -> !value.isBlank()) - .map(value -> { - switch (value) { - case "gzip": - return CompressionType.GZIP; - case "zstd": - return CompressionType.ZSTD; - default: - throw new IllegalArgumentException("Unknown compression type: " + value); - } - }) - .orElse(fallback); + .filter(value -> !value.isBlank()) + .orElse(fallback); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 9871fed5b7c..1aa4333a18c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -47,7 +47,7 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { if (isHosted()){ cluster.addComponent( new AccessLogComponent( - AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, + AccessLogComponent.AccessLogType.jsonAccessLog, "zstd", "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log", 1024,256*1024)); cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "configserver")); } else { |