diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-06-15 15:50:36 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-06-15 15:50:36 +0200 |
commit | a280e9586033270d8cada08bb90983cadfa9118a (patch) | |
tree | 8cbf03295d3836b2db6361ced31c4a44f545bb3f /config-model/src | |
parent | d1e1926fc68e3367eaf8991b8bef7975b2ac799d (diff) | |
parent | 12bfebf0f28f9fb8e2c427828e93aa9a50101066 (diff) |
Merge branch 'master' into balder/add-shared-string-repo-control
Diffstat (limited to 'config-model/src')
10 files changed, 33 insertions, 27 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java index 3209fd1703d..9c6c446b82d 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java @@ -19,9 +19,11 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * @author baldersheim */ public class DictionaryProcessor extends Processor { + public DictionaryProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } + @Override public void process(boolean validate, boolean documentsOnly) { for (SDField field : schema.allConcreteFields()) { @@ -51,4 +53,5 @@ public class DictionaryProcessor extends Processor { } } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java index c87eeab23a5..d5a0b97a416 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.PortAllocBridge; @@ -16,6 +17,7 @@ public class Logserver extends AbstractService { private static final long serialVersionUID = 1L; private static final String logArchiveDir = "$ROOT/logs/vespa/logarchive"; + private String compressionType = "gzip"; public Logserver(AbstractConfigProducer parent) { super(parent, "logserver"); @@ -27,6 +29,12 @@ public class Logserver extends AbstractService { setProp("clustername", "admin"); } + @Override + public void initService(DeployState deployState) { + super.initService(deployState); + this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm("gzip"); + } + /** * @return the startup command for the logserver */ @@ -44,6 +52,8 @@ public class Logserver extends AbstractService { sb.append("-Dlogserver.rpcListenPort=").append(getRelativePort(0)); sb.append(" "); sb.append("-Dlogserver.logarchive.dir=" + logArchiveDir); + sb.append(" "); + sb.append("-Dlogserver.logarchive.compression=" + compressionType); return sb.toString(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index 251ef48c9f7..62e99576c95 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -23,8 +23,9 @@ public class LogserverContainer extends Container { super(parent, "" + 0, 0, deployState); if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) useDynamicPorts(); LogserverContainerCluster cluster = (LogserverContainerCluster) parent; - addComponent(new AccessLogComponent( - cluster, AccessLogType.jsonAccessLog, CompressionType.GZIP, Optional.of(cluster.getName()), true)); + addComponent(new AccessLogComponent(cluster, AccessLogType.jsonAccessLog, + deployState.featureFlags().logFileCompressionAlgorithm("zstd"), + Optional.of(cluster.getName()), true)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 45e9b3a905a..f303da6c9f0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -66,7 +66,7 @@ public class ClusterControllerContainer extends Container implements "/cluster/v2/*", CLUSTERCONTROLLER_BUNDLE); addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, - AccessLogComponent.CompressionType.GZIP, + deployState.featureFlags().logFileCompressionAlgorithm("zstd"), Optional.of("controller"), deployState.isHosted())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index f159362d750..03bdf4eb12a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -59,7 +59,7 @@ public class MetricsProxyContainer extends Container implements setProp("index", String.valueOf(index)); addNodeSpecificComponents(); addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, - AccessLogComponent.CompressionType.ZSTD, + "zstd", Optional.of("metrics-proxy"), deployState.isHosted())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 79412368eae..4755f674f69 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -146,6 +146,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> private final boolean isHostedVespa; private final boolean zooKeeperLocalhostAffinity; private final int numAvailableProcessors; + private final String compressionType; private final Map<String, String> concreteDocumentTypes = new LinkedHashMap<>(); @@ -166,7 +167,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); this.zooKeeperLocalhostAffinity = zooKeeperLocalhostAffinity; - numAvailableProcessors = deployState.featureFlags().availableProcessors(); + this.numAvailableProcessors = deployState.featureFlags().availableProcessors(); + this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm("zstd"); componentGroup = new ComponentGroup<>(this, "component"); @@ -282,7 +284,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> container.setOwner(this); container.setClusterName(name); container.setProp("clustername", name) - .setProp("index", this.containers.size()); + .setProp("index", this.containers.size()) + .setProp("clustertype", "container"); containers.add(container); } @@ -526,7 +529,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> } public void addDefaultSearchAccessLog() { - var compressionType = isHostedVespa ? AccessLogComponent.CompressionType.ZSTD : AccessLogComponent.CompressionType.GZIP; // In hosted Vespa with one application container per node we do not use the container name to distinguish log files Optional<String> clusterName = isHostedVespa ? Optional.empty() : Optional.of(getName()); addComponent(new AccessLogComponent(this, AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, clusterName, isHostedVespa)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index cc0415a7630..79c108cd867 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -32,7 +32,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL private final int queueSize; private final Integer bufferSize; - public AccessLogComponent(ContainerCluster<?> cluster, AccessLogType logType, CompressionType compressionType, Optional<String> clusterName, boolean isHostedVespa) + public AccessLogComponent(ContainerCluster<?> cluster, AccessLogType logType, String compressionType, Optional<String> clusterName, boolean isHostedVespa) { // In hosted Vespa we do not use the clusterName when setting up application ContainerCluster logging this(logType, @@ -55,7 +55,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL } public AccessLogComponent(AccessLogType logType, - CompressionType compressionType, + String compressionType, String fileNamePattern, String rotationInterval, Boolean compressOnRotation, @@ -70,7 +70,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.compression = compressOnRotation; this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; - this.compressionType = compressionType; + this.compressionType = "zstd".equals(compressionType) ? CompressionType.ZSTD :CompressionType.GZIP; this.queueSize = (queueSize == null) ? 256 : queueSize; this.bufferSize = bufferSize; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index b939109dab1..3462fb8bcfc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -54,9 +54,10 @@ public class AccessLogBuilder { @Override protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element spec) { + String fallback = deployState.featureFlags().logFileCompressionAlgorithm("zstd"); return new AccessLogComponent( accessLogType, - compressionType(spec, isHostedVespa), + compressionType(spec, fallback), fileNamePattern(spec), rotationInterval(spec), compressOnRotation(spec), @@ -93,21 +94,10 @@ public class AccessLogBuilder { return nullIfEmpty(spec.getAttribute("fileNamePattern")); } - private static CompressionType compressionType(Element spec, boolean isHostedVespa) { - CompressionType fallback = isHostedVespa ? CompressionType.ZSTD : CompressionType.GZIP; + private static String compressionType(Element spec, String fallback) { return Optional.ofNullable(spec.getAttribute("compressionType")) - .filter(value -> !value.isBlank()) - .map(value -> { - switch (value) { - case "gzip": - return CompressionType.GZIP; - case "zstd": - return CompressionType.ZSTD; - default: - throw new IllegalArgumentException("Unknown compression type: " + value); - } - }) - .orElse(fallback); + .filter(value -> !value.isBlank()) + .orElse(fallback); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 9871fed5b7c..1aa4333a18c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -47,7 +47,7 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { if (isHosted()){ cluster.addComponent( new AccessLogComponent( - AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, + AccessLogComponent.AccessLogType.jsonAccessLog, "zstd", "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log", 1024,256*1024)); cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "configserver")); } else { diff --git a/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java index 1956b87a689..45a546259ae 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java @@ -62,7 +62,7 @@ public class DictionaryTestCase { Schema verifyDictionaryControl(Dictionary.Type expected, String type, String ... cfg) throws ParseException { String def = TestUtil.joinLines( - "search test {", + "schema test {", " document test {", " field n1 type " + type + " {", " indexing: summary | attribute", |