From ba9bfeea242c9151468b2f2d599ffc9ede5dfd5b Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 14 Jun 2022 10:27:02 +0200 Subject: Set property clustertype for container cluster --- .../main/java/com/yahoo/vespa/model/container/ContainerCluster.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'config-model/src') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 79412368eae..69e05ea15c9 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -282,7 +282,8 @@ public abstract class ContainerCluster container.setOwner(this); container.setClusterName(name); container.setProp("clustername", name) - .setProp("index", this.containers.size()); + .setProp("index", this.containers.size()) + .setProp("clustertype", "container"); containers.add(container); } -- cgit v1.2.3 From e78543c0bdc39dd40f1db348ccf34812a0b20133 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Tue, 14 Jun 2022 11:51:13 +0000 Subject: use new feature flag for compression format * note: all defaults should remain the same in this commit --- .../java/com/yahoo/vespa/model/admin/Logserver.java | 10 ++++++++++ .../yahoo/vespa/model/admin/LogserverContainer.java | 5 +++-- .../ClusterControllerContainer.java | 2 +- .../admin/metricsproxy/MetricsProxyContainer.java | 2 +- .../vespa/model/container/ContainerCluster.java | 5 +++-- .../container/component/AccessLogComponent.java | 6 +++--- .../vespa/model/container/xml/AccessLogBuilder.java | 20 +++++--------------- .../xml/ConfigServerContainerModelBuilder.java | 2 +- 8 files changed, 27 insertions(+), 25 deletions(-) (limited to 'config-model/src') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java index c87eeab23a5..d5a0b97a416 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.PortAllocBridge; @@ -16,6 +17,7 @@ public class Logserver extends AbstractService { private static final long serialVersionUID = 1L; private static final String logArchiveDir = "$ROOT/logs/vespa/logarchive"; + private String compressionType = "gzip"; public Logserver(AbstractConfigProducer parent) { super(parent, "logserver"); @@ -27,6 +29,12 @@ public class Logserver extends AbstractService { setProp("clustername", "admin"); } + @Override + public void initService(DeployState deployState) { + super.initService(deployState); + this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm("gzip"); + } + /** * @return the startup command for the logserver */ @@ -44,6 +52,8 @@ public class Logserver extends AbstractService { sb.append("-Dlogserver.rpcListenPort=").append(getRelativePort(0)); sb.append(" "); sb.append("-Dlogserver.logarchive.dir=" + logArchiveDir); + sb.append(" "); + sb.append("-Dlogserver.logarchive.compression=" + compressionType); return sb.toString(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index 251ef48c9f7..e4004cc221f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -23,8 +23,9 @@ public class LogserverContainer extends Container { super(parent, "" + 0, 0, deployState); if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) useDynamicPorts(); LogserverContainerCluster cluster = (LogserverContainerCluster) parent; - addComponent(new AccessLogComponent( - cluster, AccessLogType.jsonAccessLog, CompressionType.GZIP, Optional.of(cluster.getName()), true)); + addComponent(new AccessLogComponent(cluster, AccessLogType.jsonAccessLog, + deployState.featureFlags().logFileCompressionAlgorithm("gzip"), + Optional.of(cluster.getName()), true)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 45e9b3a905a..48d11a078e0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -66,7 +66,7 @@ public class ClusterControllerContainer extends Container implements "/cluster/v2/*", CLUSTERCONTROLLER_BUNDLE); addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, - AccessLogComponent.CompressionType.GZIP, + deployState.featureFlags().logFileCompressionAlgorithm("gzip"), Optional.of("controller"), deployState.isHosted())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index f159362d750..03bdf4eb12a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -59,7 +59,7 @@ public class MetricsProxyContainer extends Container implements setProp("index", String.valueOf(index)); addNodeSpecificComponents(); addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, - AccessLogComponent.CompressionType.ZSTD, + "zstd", Optional.of("metrics-proxy"), deployState.isHosted())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 79412368eae..405cda4494c 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -146,6 +146,7 @@ public abstract class ContainerCluster private final boolean isHostedVespa; private final boolean zooKeeperLocalhostAffinity; private final int numAvailableProcessors; + private final String compressionType; private final Map concreteDocumentTypes = new LinkedHashMap<>(); @@ -166,7 +167,8 @@ public abstract class ContainerCluster this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); this.zooKeeperLocalhostAffinity = zooKeeperLocalhostAffinity; - numAvailableProcessors = deployState.featureFlags().availableProcessors(); + this.numAvailableProcessors = deployState.featureFlags().availableProcessors(); + this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm(isHostedVespa ? "zstd" : "gzip"); componentGroup = new ComponentGroup<>(this, "component"); @@ -526,7 +528,6 @@ public abstract class ContainerCluster } public void addDefaultSearchAccessLog() { - var compressionType = isHostedVespa ? AccessLogComponent.CompressionType.ZSTD : AccessLogComponent.CompressionType.GZIP; // In hosted Vespa with one application container per node we do not use the container name to distinguish log files Optional clusterName = isHostedVespa ? Optional.empty() : Optional.of(getName()); addComponent(new AccessLogComponent(this, AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, clusterName, isHostedVespa)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index cc0415a7630..79c108cd867 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -32,7 +32,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL private final int queueSize; private final Integer bufferSize; - public AccessLogComponent(ContainerCluster cluster, AccessLogType logType, CompressionType compressionType, Optional clusterName, boolean isHostedVespa) + public AccessLogComponent(ContainerCluster cluster, AccessLogType logType, String compressionType, Optional clusterName, boolean isHostedVespa) { // In hosted Vespa we do not use the clusterName when setting up application ContainerCluster logging this(logType, @@ -55,7 +55,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL } public AccessLogComponent(AccessLogType logType, - CompressionType compressionType, + String compressionType, String fileNamePattern, String rotationInterval, Boolean compressOnRotation, @@ -70,7 +70,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.compression = compressOnRotation; this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; - this.compressionType = compressionType; + this.compressionType = "zstd".equals(compressionType) ? CompressionType.ZSTD :CompressionType.GZIP; this.queueSize = (queueSize == null) ? 256 : queueSize; this.bufferSize = bufferSize; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index b939109dab1..bd2627f29d0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -54,9 +54,10 @@ public class AccessLogBuilder { @Override protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element spec) { + String fallback = deployState.featureFlags().logFileCompressionAlgorithm(isHostedVespa ? "zstd" : "gzip"); return new AccessLogComponent( accessLogType, - compressionType(spec, isHostedVespa), + compressionType(spec, fallback), fileNamePattern(spec), rotationInterval(spec), compressOnRotation(spec), @@ -93,21 +94,10 @@ public class AccessLogBuilder { return nullIfEmpty(spec.getAttribute("fileNamePattern")); } - private static CompressionType compressionType(Element spec, boolean isHostedVespa) { - CompressionType fallback = isHostedVespa ? CompressionType.ZSTD : CompressionType.GZIP; + private static String compressionType(Element spec, String fallback) { return Optional.ofNullable(spec.getAttribute("compressionType")) - .filter(value -> !value.isBlank()) - .map(value -> { - switch (value) { - case "gzip": - return CompressionType.GZIP; - case "zstd": - return CompressionType.ZSTD; - default: - throw new IllegalArgumentException("Unknown compression type: " + value); - } - }) - .orElse(fallback); + .filter(value -> !value.isBlank()) + .orElse(fallback); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 9871fed5b7c..1aa4333a18c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -47,7 +47,7 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { if (isHosted()){ cluster.addComponent( new AccessLogComponent( - AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, + AccessLogComponent.AccessLogType.jsonAccessLog, "zstd", "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log", 1024,256*1024)); cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "configserver")); } else { -- cgit v1.2.3 From 92e6cf7365aa33a8405aee20924b42010247bc4e Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Wed, 15 Jun 2022 07:09:20 +0000 Subject: common default compression type --- .../src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java | 2 +- .../main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'config-model/src') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 405cda4494c..83112d14ba4 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -168,7 +168,7 @@ public abstract class ContainerCluster this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); this.zooKeeperLocalhostAffinity = zooKeeperLocalhostAffinity; this.numAvailableProcessors = deployState.featureFlags().availableProcessors(); - this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm(isHostedVespa ? "zstd" : "gzip"); + this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm("zstd"); componentGroup = new ComponentGroup<>(this, "component"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index bd2627f29d0..3462fb8bcfc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -54,7 +54,7 @@ public class AccessLogBuilder { @Override protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element spec) { - String fallback = deployState.featureFlags().logFileCompressionAlgorithm(isHostedVespa ? "zstd" : "gzip"); + String fallback = deployState.featureFlags().logFileCompressionAlgorithm("zstd"); return new AccessLogComponent( accessLogType, compressionType(spec, fallback), -- cgit v1.2.3 From 72d23eab3d9f0f370d73fffa836f9b291bc8af32 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Wed, 15 Jun 2022 07:55:42 +0000 Subject: use zstd for more access logs --- .../src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java | 2 +- .../vespa/model/admin/clustercontroller/ClusterControllerContainer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'config-model/src') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index e4004cc221f..62e99576c95 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -24,7 +24,7 @@ public class LogserverContainer extends Container { if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) useDynamicPorts(); LogserverContainerCluster cluster = (LogserverContainerCluster) parent; addComponent(new AccessLogComponent(cluster, AccessLogType.jsonAccessLog, - deployState.featureFlags().logFileCompressionAlgorithm("gzip"), + deployState.featureFlags().logFileCompressionAlgorithm("zstd"), Optional.of(cluster.getName()), true)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 48d11a078e0..f303da6c9f0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -66,7 +66,7 @@ public class ClusterControllerContainer extends Container implements "/cluster/v2/*", CLUSTERCONTROLLER_BUNDLE); addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, - deployState.featureFlags().logFileCompressionAlgorithm("gzip"), + deployState.featureFlags().logFileCompressionAlgorithm("zstd"), Optional.of("controller"), deployState.isHosted())); -- cgit v1.2.3 From 188999a50a45f11a4bbe8f56fee1a8ecabb4b97f Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 15 Jun 2022 13:37:03 +0200 Subject: No functional changes --- .../main/java/com/yahoo/schema/processing/DictionaryProcessor.java | 3 +++ .../test/java/com/yahoo/schema/processing/DictionaryTestCase.java | 2 +- document/src/main/java/com/yahoo/document/ReferenceDataType.java | 2 +- .../java/com/yahoo/document/datatypes/ReferenceFieldValue.java | 7 +++---- 4 files changed, 8 insertions(+), 6 deletions(-) (limited to 'config-model/src') diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java index 3209fd1703d..9c6c446b82d 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java @@ -19,9 +19,11 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * @author baldersheim */ public class DictionaryProcessor extends Processor { + public DictionaryProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } + @Override public void process(boolean validate, boolean documentsOnly) { for (SDField field : schema.allConcreteFields()) { @@ -51,4 +53,5 @@ public class DictionaryProcessor extends Processor { } } } + } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java index 1956b87a689..45a546259ae 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java @@ -62,7 +62,7 @@ public class DictionaryTestCase { Schema verifyDictionaryControl(Dictionary.Type expected, String type, String ... cfg) throws ParseException { String def = TestUtil.joinLines( - "search test {", + "schema test {", " document test {", " field n1 type " + type + " {", " indexing: summary | attribute", diff --git a/document/src/main/java/com/yahoo/document/ReferenceDataType.java b/document/src/main/java/com/yahoo/document/ReferenceDataType.java index 8cf6b665acf..ee1afa086d7 100644 --- a/document/src/main/java/com/yahoo/document/ReferenceDataType.java +++ b/document/src/main/java/com/yahoo/document/ReferenceDataType.java @@ -16,7 +16,7 @@ public class ReferenceDataType extends DataType { // Magic number for Identifiable, see document/util/identifiable.h public static final int classId = registerClass(Ids.document + 68, ReferenceDataType.class); - private StructuredDataType targetType; + private final StructuredDataType targetType; public ReferenceDataType(DocumentType targetType, int id) { this((StructuredDataType)targetType, id); diff --git a/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java index b76c3f2989f..ebc39b14eb1 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java @@ -49,6 +49,7 @@ public class ReferenceFieldValue extends FieldValue { /** * Creates a reference pointing to a particular document instance in the document * type referenced by referenceType. + * * @param referenceType reference target type * @param documentId document ID of the same document type as that given by referenceType * @throws IllegalArgumentException if documentId is not of the expected document type @@ -64,7 +65,7 @@ public class ReferenceFieldValue extends FieldValue { } private static void requireIdOfMatchingType(ReferenceDataType referenceType, DocumentId id) { - final String expectedTypeName = referenceType.getTargetType().getName(); + String expectedTypeName = referenceType.getTargetType().getName(); if (!id.getDocType().equals(expectedTypeName)) { throw new IllegalArgumentException(String.format( "Can't assign document ID '%s' (of type '%s') to reference of document type '%s'", @@ -87,9 +88,7 @@ public class ReferenceFieldValue extends FieldValue { } @Override - public void printXml(XmlStream xml) { - // TODO do we need/want this? - } + public void printXml(XmlStream xml) { } @Override public void clear() { -- cgit v1.2.3