diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-10-17 16:27:02 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-10-17 16:27:02 +0200 |
commit | bbcccf78cfaa5438c18f188c5dd15a9a979617ee (patch) | |
tree | cb3b35e15c47c108bae252c1cc169945c88c365c /config-model | |
parent | 849401dd245eb9193d1ca31bc288c6b665795747 (diff) | |
parent | b7123d3a07bc823961e452ad527d00e236012ebe (diff) |
Merge branch 'master' into balder/gc-unused-phrase-flags
Diffstat (limited to 'config-model')
22 files changed, 114 insertions, 239 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index b1b6870a004..eb628db6975 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -54,8 +54,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; - private int maxConcurrentMergesPerNode = 16; - private int maxMergeQueueSize = 100; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = Collections.emptyList(); private double resourceLimitDisk = 0.75; @@ -64,15 +62,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean containerDumpHeapOnShutdownTimeout = false; private double containerShutdownTimeout = 50.0; private int maxUnCommittedMemory = 123456; - private boolean unorderedMergeChaining = true; private List<String> zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; - private String mergeThrottlingPolicy = "STATIC"; - private double persistenceThrottlingWsDecrementFactor = 1.2; - private double persistenceThrottlingWsBackoff = 0.95; - private int persistenceThrottlingWindowSize = -1; - private double persistenceThrottlingWsResizeRate = 3.0; - private boolean persistenceThrottlingOfMergeFeedOps = true; private boolean useV8GeoPositions = true; private List<String> environmentVariables = List.of(); private boolean loadCodeAsHugePages = false; @@ -117,23 +108,14 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } - @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } - @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } - @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } - @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } - @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } - @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } - @Override public int persistenceThrottlingWindowSize() { return persistenceThrottlingWindowSize; } - @Override public double persistenceThrottlingWsResizeRate() { return persistenceThrottlingWsResizeRate; } - @Override public boolean persistenceThrottlingOfMergeFeedOps() { return persistenceThrottlingOfMergeFeedOps; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List<String> environmentVariables() { return environmentVariables; } @Override public Architecture adminClusterArchitecture() { return adminClusterNodeResourcesArchitecture; } @@ -214,15 +196,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setMaxConcurrentMergesPerNode(int maxConcurrentMergesPerNode) { - this.maxConcurrentMergesPerNode = maxConcurrentMergesPerNode; - return this; - } - public TestProperties setMaxMergeQueueSize(int maxMergeQueueSize) { - this.maxMergeQueueSize = maxMergeQueueSize; - return this; - } - public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; @@ -313,11 +286,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setUnorderedMergeChaining(boolean unordered) { - unorderedMergeChaining = unordered; - return this; - } - public TestProperties setZoneDnsSuffixes(List<String> zoneDnsSuffixes) { this.zoneDnsSuffixes = List.copyOf(zoneDnsSuffixes); return this; @@ -328,36 +296,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setMergeThrottlingPolicy(String policy) { - this.mergeThrottlingPolicy = policy; - return this; - } - - public TestProperties setPersistenceThrottlingWsDecrementFactor(double factor) { - this.persistenceThrottlingWsDecrementFactor = factor; - return this; - } - - public TestProperties setPersistenceThrottlingWsBackoff(double backoff) { - this.persistenceThrottlingWsBackoff = backoff; - return this; - } - - public TestProperties setPersistenceThrottlingWindowSize(int windowSize) { - this.persistenceThrottlingWindowSize = windowSize; - return this; - } - - public TestProperties setPersistenceThrottlingWsResizeRate(double resizeRate) { - this.persistenceThrottlingWsResizeRate = resizeRate; - return this; - } - - public TestProperties setPersistenceThrottlingOfMergeFeedOps(boolean throttleOps) { - this.persistenceThrottlingOfMergeFeedOps = throttleOps; - return this; - } - public TestProperties setUseV8GeoPositions(boolean value) { this.useV8GeoPositions = value; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 9ee279c68d3..2d98c8b35c0 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -10,6 +10,7 @@ import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; @@ -85,6 +86,7 @@ public class MockApplicationPackage implements ApplicationPackage { ApplicationId.from(TenantName.defaultName(), ApplicationName.from(APPLICATION_NAME), InstanceName.defaultName()), + Tags.empty(), "checksum", APPLICATION_GENERATION, 0L); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java index 793505acd01..6f470cfdc56 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java @@ -41,19 +41,19 @@ public class PagedAttributeValidator extends Processor { private void validatePagedSetting(Field field, Attribute attribute) { if (!isSupportedType(attribute)) { - fail(schema, field, "The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types"); + fail(schema, field, "The 'paged' attribute setting is not supported for fast-rank tensor and predicate types"); } } private boolean isSupportedType(Attribute attribute) { var type = attribute.getType(); return (type != Attribute.Type.PREDICATE) && - (isSupportedTensorType(attribute.tensorType())); + (isSupportedTensorType(attribute.tensorType(), attribute.isFastRank())); } - private boolean isSupportedTensorType(Optional<TensorType> tensorType) { + private boolean isSupportedTensorType(Optional<TensorType> tensorType, boolean fastRank) { if (tensorType.isPresent()) { - return isDenseTensorType(tensorType.get()); + return !fastRank; } return true; } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index e94518c988d..d1d22886642 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -37,33 +37,20 @@ public enum SummaryTransform { /** Returns the bolded version of this transform if possible, throws if not */ public SummaryTransform bold() { - switch (this) { - case NONE: - case BOLDED: - return BOLDED; - - case DYNAMICBOLDED: - case DYNAMICTEASER: - return DYNAMICBOLDED; - - default: - throw new IllegalArgumentException("Can not bold a '" + this + "' field."); - } + return switch (this) { + case NONE, BOLDED -> BOLDED; + case DYNAMICBOLDED, DYNAMICTEASER -> DYNAMICBOLDED; + default -> throw new IllegalArgumentException("Can not bold a '" + this + "' field."); + }; } /** Returns the unbolded version of this transform */ public SummaryTransform unbold() { - switch (this) { - case NONE: - case BOLDED: - return NONE; - - case DYNAMICBOLDED: - return DYNAMICTEASER; - - default: - return this; - } + return switch (this) { + case NONE, BOLDED -> NONE; + case DYNAMICBOLDED -> DYNAMICTEASER; + default -> this; + }; } /** Returns whether this value is bolded */ @@ -83,23 +70,16 @@ public enum SummaryTransform { /** Returns whether this transform always gets its value by accessing memory only */ public boolean isInMemory() { - switch (this) { - case ATTRIBUTE: - case DISTANCE: - case POSITIONS: - case GEOPOS: - case RANKFEATURES: - case SUMMARYFEATURES: - case ATTRIBUTECOMBINER: - case MATCHED_ATTRIBUTE_ELEMENTS_FILTER: - return true; - - default: - return false; - } + return switch (this) { + case ATTRIBUTE, DISTANCE, POSITIONS, GEOPOS, RANKFEATURES, SUMMARYFEATURES, ATTRIBUTECOMBINER, MATCHED_ATTRIBUTE_ELEMENTS_FILTER -> + true; + default -> false; + }; } + @Override public String toString() { return name; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index f01e7799e13..4d03944c4e9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -202,6 +202,14 @@ public class VespaMetricSet { metrics.add(new Metric("jdisc.deactivated_containers.total.last")); metrics.add(new Metric("jdisc.deactivated_containers.with_retained_refs.last")); + metrics.add(new Metric("jdisc.singleton.is_active.last")); + metrics.add(new Metric("jdisc.singleton.activation.count.last")); + metrics.add(new Metric("jdisc.singleton.activation.failure.count.last")); + metrics.add(new Metric("jdisc.singleton.activation.millis.last")); + metrics.add(new Metric("jdisc.singleton.deactivation.count.last")); + metrics.add(new Metric("jdisc.singleton.deactivation.failure.count.last")); + metrics.add(new Metric("jdisc.singleton.deactivation.millis.last")); + metrics.add(new Metric("athenz-tenant-cert.expiry.seconds.last")); metrics.add(new Metric("container-iam-role.expiry.seconds")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 4c74282c061..06453bffaaf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index e316f826ad6..8907c21d39a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container; import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler; +import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; @@ -305,6 +306,15 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } + @Override + public void getConfig(CuratorConfig.Builder builder) { + if (getParent() instanceof ConfigserverCluster) return; // Produces its own config + super.getConfig(builder); + + // 12s is 2x the current ZookeeperServerConfig.tickTime() of 6s, and the default minimum the server will accept. + builder.zookeeperSessionTimeoutSeconds(12); // TODO jonmv: make configurable + } + public Optional<String> getTlsClientAuthority() { return tlsClientAuthority; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 5e09dd4732d..307e0e17955 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -246,6 +246,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ", have " + nonRetiredNodes + " non-retired"); } cluster.addSimpleComponent("com.yahoo.vespa.curator.Curator", null, "zkfacade"); + cluster.addSimpleComponent("com.yahoo.vespa.curator.CuratorWrapper", null, "zkfacade"); // These need to be setup so that they will use the container's config id, since each container // have different config (id of zookeeper server) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index b1258247e2e..dae823bcc9f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -32,7 +32,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final GcOptions gc; private final boolean hasIndexedDocumentType; private final int maxActivationInhibitedOutOfSyncGroups; - private final boolean unorderedMergeChaining; private final boolean useTwoPhaseDocumentGc; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -95,20 +94,18 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl final GcOptions gc = parseGcOptions(documentsNode); final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); - boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining(); boolean useTwoPhaseDocumentGc = deployState.getProperties().featureFlags().useTwoPhaseDocumentGc(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, - maxInhibitedGroups, unorderedMergeChaining, useTwoPhaseDocumentGc); + maxInhibitedGroups, useTwoPhaseDocumentGc); } } private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, GcOptions gc, boolean hasIndexedDocumentType, int maxActivationInhibitedOutOfSyncGroups, - boolean unorderedMergeChaining, boolean useTwoPhaseDocumentGc) { super(parent, "distributor"); @@ -117,7 +114,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl this.gc = gc; this.hasIndexedDocumentType = hasIndexedDocumentType; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; - this.unorderedMergeChaining = unorderedMergeChaining; this.useTwoPhaseDocumentGc = useTwoPhaseDocumentGc; } @@ -131,7 +127,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.enable_revert(parent.getPersistence().supportRevert()); builder.disable_bucket_activation(!hasIndexedDocumentType); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); - builder.use_unordered_merge_chaining(unorderedMergeChaining); builder.enable_two_phase_garbage_collection(useTwoPhaseDocumentGc); bucketSplitting.getConfig(builder); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index ff905187969..092b94d72dc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -46,11 +46,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final ContentCluster cluster; private final int responseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; - private final double persistenceThrottlingWsDecrementFactor; - private final double persistenceThrottlingWsBackoff; - private final int persistenceThrottlingWindowSize; - private final double persistenceThrottlingWsResizeRate; - private final boolean persistenceThrottlingOfMergeFeedOps; private final boolean useAsyncMessageHandlingOnSchedule; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { @@ -66,11 +61,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { this.cluster = parent; this.responseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); - this.persistenceThrottlingWsDecrementFactor = featureFlags.persistenceThrottlingWsDecrementFactor(); - this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff(); - this.persistenceThrottlingWindowSize = featureFlags.persistenceThrottlingWindowSize(); - this.persistenceThrottlingWsResizeRate = featureFlags.persistenceThrottlingWsResizeRate(); - this.persistenceThrottlingOfMergeFeedOps = featureFlags.persistenceThrottlingOfMergeFeedOps(); this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); } @@ -84,14 +74,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); - throttleBuilder.window_size_decrement_factor(persistenceThrottlingWsDecrementFactor); - throttleBuilder.window_size_backoff(persistenceThrottlingWsBackoff); - if (persistenceThrottlingWindowSize > 0) { - throttleBuilder.min_window_size(persistenceThrottlingWindowSize); - throttleBuilder.max_window_size(persistenceThrottlingWindowSize); - } - throttleBuilder.resize_rate(persistenceThrottlingWsResizeRate); - throttleBuilder.throttle_individual_merge_feed_ops(persistenceThrottlingOfMergeFeedOps); builder.async_operation_throttler(throttleBuilder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index e66f2c48f26..4298488b1fd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content.storagecluster; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -11,10 +10,10 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster; */ public class StorServerProducer implements StorServerConfig.Producer { public static class Builder { - StorServerProducer build(ModelContext.Properties properties, ModelElement element) { + StorServerProducer build(ModelElement element) { ModelElement tuning = element.child("tuning"); - StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element), properties.featureFlags()); + StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element)); if (tuning == null) return producer; ModelElement merges = tuning.child("merges"); @@ -29,7 +28,6 @@ public class StorServerProducer implements StorServerConfig.Producer { private final String clusterName; private Integer maxMergesPerNode; private Integer queueSize; - private final StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -44,19 +42,8 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } - private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) { - try { - return StorServerConfig.Merge_throttling_policy.Type.Enum.valueOf(policyType); - } catch (Throwable t) { - return StorServerConfig.Merge_throttling_policy.Type.STATIC; - } - } - - StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { + StorServerProducer(String clusterName) { this.clusterName = clusterName; - maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); - queueSize = featureFlags.maxMergeQueueSize(); - mergeThrottlingPolicyType = toThrottlePolicyType(featureFlags.mergeThrottlingPolicy()); } @Override @@ -73,7 +60,5 @@ public class StorServerProducer implements StorServerConfig.Producer { if (queueSize != null) { builder.max_merge_queue_size(queueSize); } - // TODO set throttle policy params based on existing or separate flags - builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index da82a69842a..9b59f6db742 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -38,7 +38,7 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> ContentCluster.getClusterId(clusterElem), new FileStorProducer.Builder().build(deployState.getProperties(), cluster, clusterElem), new IntegrityCheckerProducer.Builder().build(cluster, clusterElem), - new StorServerProducer.Builder().build(deployState.getProperties(), clusterElem), + new StorServerProducer.Builder().build(clusterElem), new StorVisitorProducer.Builder().build(clusterElem), new PersistenceProducer.Builder().build(clusterElem)); } diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 9723e531bd2..bf94ee3b750 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -35,6 +35,7 @@ PrimitiveStep = Instance = element instance { attribute id { xsd:string } & + attribute tags { xsd:string }? & attribute athenz-service { xsd:string }? & attribute cloud-account { xsd:string }? & StepExceptInstance diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index c1dd62316db..6507341670f 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Tags; import com.yahoo.document.DataType; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -225,17 +226,19 @@ public class ApplicationDeployTest { IOUtils.copyDirectory(new File(appPkg), tmp); ApplicationId applicationId = ApplicationId.from("tenant1", "application1", "instance1"); DeployData deployData = new DeployData("bar", - applicationId, - 13L, - false, - 1337L, - 3L); + applicationId, + Tags.fromString("tag1 tag2"), + 13L, + false, + 1337L, + 3L); FilesApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(tmp, deployData); app.writeMetaData(); FilesApplicationPackage newApp = FilesApplicationPackage.fromFileWithDeployData(tmp, deployData); ApplicationMetaData meta = newApp.getMetaData(); assertEquals("bar", meta.getDeployPath()); assertEquals(applicationId, meta.getApplicationId()); + assertEquals(Tags.fromString("tag1 tag2"), meta.getTags()); assertEquals(13L, (long) meta.getDeployTimestamp()); assertEquals(1337L, (long) meta.getGeneration()); assertEquals(3L, meta.getPreviousActiveGeneration()); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java index 719db2ffdcc..9de44d28c09 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java @@ -71,8 +71,13 @@ public class PagedAttributeValidatorTestCase { } @Test - void non_dense_tensor_attribute_does_not_support_paged_setting() throws ParseException { - assertPagedSettingNotSupported("tensor(x{},y[2])"); + void non_dense_none_fast_rank_tensor_attribute_supports_paged_setting() throws ParseException { + assertPagedSupported("tensor(x{},y[2])"); + } + + @Test + void non_dense_fast_rank_tensor_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("tensor(x{},y[2])", true); } @Test @@ -82,36 +87,45 @@ public class PagedAttributeValidatorTestCase { @Test void reference_attribute_support_paged_setting() throws ParseException { - assertPagedSupported("reference<parent>", Optional.of(getSd("parent", "int"))); + assertPagedSupported("reference<parent>", Optional.of(getSd("parent", "int", false))); } private void assertPagedSettingNotSupported(String fieldType) throws ParseException { - assertPagedSettingNotSupported(fieldType, Optional.empty()); + assertPagedSettingNotSupported(fieldType, false); + } + + private void assertPagedSettingNotSupported(String fieldType, boolean fastRank) throws ParseException { + assertPagedSettingNotSupported(fieldType, fastRank, Optional.empty()); } - private void assertPagedSettingNotSupported(String fieldType, Optional<String> parentSd) throws ParseException { + private void assertPagedSettingNotSupported(String fieldType, boolean fastRank, Optional<String> parentSd) throws ParseException { try { if (parentSd.isPresent()) { - createFromStrings(new BaseDeployLogger(), parentSd.get(), getSd(fieldType)); + createFromStrings(new BaseDeployLogger(), parentSd.get(), getSd(fieldType, fastRank)); } else { - createFromString(getSd(fieldType)); + createFromString(getSd(fieldType, fastRank)); } fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'foo': The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types", + assertEquals("For schema 'test', field 'foo': The 'paged' attribute setting is not supported for fast-rank tensor and predicate types", e.getMessage()); } } private String getSd(String fieldType) { - return getSd("test", fieldType); + return getSd(fieldType, false); + } + + private String getSd(String fieldType, boolean fastRank) { + return getSd("test", fieldType, fastRank); } - private String getSd(String docType, String fieldType) { + private String getSd(String docType, String fieldType, boolean fastRank) { return joinLines( "schema " + docType + " {", " document " + docType + " {", " field foo type " + fieldType + "{", + (fastRank ? "attribute: fast-rank" : ""), " indexing: attribute", " attribute: paged", " }", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index 376cf49c396..eb77187014f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -413,6 +413,11 @@ public class ClusterControllerTestCase extends DomBuilderTest { assertEquals(0, qrStartConfig.jvm().directMemorySizeCache()); assertEquals(16, qrStartConfig.jvm().baseMaxDirectMemorySize()); + CuratorConfig.Builder curatorBuilder = new CuratorConfig.Builder(); + model.getConfig(curatorBuilder, "foo"); + CuratorConfig curatorConfig = curatorBuilder.build(); + assertEquals(120, curatorConfig.zookeeperSessionTimeoutSeconds()); + assertReindexingConfigPresent(model); assertReindexingConfiguredOnAdminCluster(model); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index cc6b84de698..da70daa2b4d 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -343,6 +343,7 @@ public class ContainerClusterTest { assertEquals(List.of("host-c1", "host-c2"), config.server().stream().map(CuratorConfig.Server::hostname).collect(Collectors.toList())); assertTrue(config.zookeeperLocalhostAffinity()); + assertEquals(12, config.zookeeperSessionTimeoutSeconds()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 272dfa19f64..cf6b6365792 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -570,6 +570,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { ApplicationContainerCluster cluster = model.getContainerClusters().get("default"); assertNotNull(cluster); assertComponentConfigured(cluster, "com.yahoo.vespa.curator.Curator"); + assertComponentConfigured(cluster, "com.yahoo.vespa.curator.CuratorWrapper"); cluster.getContainers().forEach(container -> { assertComponentConfigured(container, "com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer"); assertComponentConfigured(container, "com.yahoo.vespa.zookeeper.Reconfigurer"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 0ba47d9c58b..c41840eaefa 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1132,23 +1132,6 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(4, resolveTunedNumDistributorStripesConfig(65)); } - @Test - void unordered_merge_chaining_config_controlled_by_properties() throws Exception { - assertFalse(resolveUnorderedMergeChainingConfig(Optional.of(false))); - assertTrue(resolveUnorderedMergeChainingConfig(Optional.empty())); - } - - private boolean resolveUnorderedMergeChainingConfig(Optional<Boolean> unorderedMergeChaining) throws Exception { - var props = new TestProperties(); - if (unorderedMergeChaining.isPresent()) { - props.setUnorderedMergeChaining(unorderedMergeChaining.get()); - } - var cluster = createOneNodeCluster(props); - var builder = new StorDistributormanagerConfig.Builder(); - cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).use_unordered_merge_chaining(); - } - private boolean resolveTwoPhaseGcConfigWithFeatureFlag(Boolean flagEnableTwoPhase) { var props = new TestProperties(); if (flagEnableTwoPhase != null) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index f7afcc281f9..f3a59733ece 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -120,9 +120,7 @@ public class StorageClusterTest { parse(cluster("foofighters", joinLines( "<tuning>", " <merges max-per-node=\"1K\" max-queue-size=\"10K\"/>", - "</tuning>")), - new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37) - ).getConfig(builder); + "</tuning>"))).getConfig(builder); StorServerConfig config = new StorServerConfig(builder); assertEquals(1024, config.max_merges_per_node()); @@ -174,9 +172,9 @@ public class StorageClusterTest { @Test void testMergeFeatureFlags() { - var config = configFromProperties(new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37)); - assertEquals(37, config.max_merges_per_node()); - assertEquals(1919, config.max_merge_queue_size()); + var config = configFromProperties(new TestProperties()); + assertEquals(16, config.max_merges_per_node()); + assertEquals(100, config.max_merge_queue_size()); } @Test @@ -186,19 +184,6 @@ public class StorageClusterTest { } @Test - void merge_throttling_policy_config_is_derived_from_flag() { - var config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("STATIC")); - assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); - - config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("DYNAMIC")); - assertEquals(StorServerConfig.Merge_throttling_policy.Type.DYNAMIC, config.merge_throttling_policy().type()); - - // Invalid enum values fall back to the default - config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("UKULELE")); - assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); - } - - @Test void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); parse(cluster("bees", @@ -340,23 +325,6 @@ public class StorageClusterTest { } @Test - void persistence_dynamic_throttling_parameters_can_be_set_through_feature_flags() { - var config = filestorConfigFromProducer(simpleCluster(new TestProperties() - .setPersistenceThrottlingWsDecrementFactor(1.5) - .setPersistenceThrottlingWsBackoff(0.8) - .setPersistenceThrottlingWindowSize(42) - .setPersistenceThrottlingWsResizeRate(2.5) - .setPersistenceThrottlingOfMergeFeedOps(false))); - assertEquals(1.5, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); - assertEquals(0.8, config.async_operation_throttler().window_size_backoff(), 0.0001); - // If window size is set, min and max are locked to the same value - assertEquals(42, config.async_operation_throttler().min_window_size()); - assertEquals(42, config.async_operation_throttler().max_window_size()); - assertEquals(2.5, config.async_operation_throttler().resize_rate(), 0.0001); - assertFalse(config.async_operation_throttler().throttle_individual_merge_feed_ops()); - } - - @Test void integrity_checker_explicitly_disabled_when_not_running_with_vds_provider() { StorIntegritycheckerConfig.Builder builder = new StorIntegritycheckerConfig.Builder(); parse(cluster("bees", "")).getConfig(builder); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 8830e5484b3..ee885cdf43e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.search.IndexedSearchCluster; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -55,7 +54,7 @@ public class DocumentDatabaseTestCase { @Test void requireThatMixedModeConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 1.0); + verifyConcurrency(List.of(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 1.0); } @Test @@ -63,7 +62,7 @@ public class DocumentDatabaseTestCase { String feedTuning = "<feeding>" + " <concurrency>0.7</concurrency>" + "</feeding>\n"; - verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), feedTuning, 0.7); + verifyConcurrency(List.of(DocType.create("a", "index"), DocType.create("b", "streaming")), feedTuning, 0.7); } @Test @@ -77,11 +76,11 @@ public class DocumentDatabaseTestCase { } private void verifyConcurrency(String mode, String xmlTuning, double expectedConcurrency, double featureFlagConcurrency) { - verifyConcurrency(Arrays.asList(DocType.create("a", mode)), xmlTuning, expectedConcurrency, featureFlagConcurrency); + verifyConcurrency(List.of(DocType.create("a", mode)), xmlTuning, expectedConcurrency, featureFlagConcurrency); } private void verifyConcurrency(String mode, String xmlTuning, double expectedConcurrency) { - verifyConcurrency(Arrays.asList(DocType.create("a", mode)), xmlTuning, expectedConcurrency, null); + verifyConcurrency(List.of(DocType.create("a", mode)), xmlTuning, expectedConcurrency, null); } private void verifyConcurrency(List<DocType> nameAndModes, String xmlTuning, double expectedConcurrency) { @@ -114,14 +113,14 @@ public class DocumentDatabaseTestCase { @Test void requireFeedNicenessIsReflected() { - verifyFeedNiceness(Arrays.asList(DocType.create("a", "index")), 0.0, null); - verifyFeedNiceness(Arrays.asList(DocType.create("a", "index")), 0.32, 0.32); + verifyFeedNiceness(List.of(DocType.create("a", "index")), 0.0, null); + verifyFeedNiceness(List.of(DocType.create("a", "index")), 0.32, 0.32); } @Test void requireThatModeIsSet() { var tester = new SchemaTester(); - VespaModel model = tester.createModel(Arrays.asList(DocType.create("a", "index"), + VespaModel model = tester.createModel(List.of(DocType.create("a", "index"), DocType.create("b", "streaming"), DocType.create("c", "store-only")), ""); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); @@ -150,8 +149,8 @@ public class DocumentDatabaseTestCase { @Test void requireThatMixedModeInitialDocumentCountIsReflectedCorrectlyForDefault() { final long DEFAULT = 1024L; - verifyInitialDocumentCount(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), - "", Arrays.asList(DEFAULT, DEFAULT)); + verifyInitialDocumentCount(List.of(DocType.create("a", "index"), DocType.create("b", "streaming")), + "", List.of(DEFAULT, DEFAULT)); } @Test @@ -160,8 +159,8 @@ public class DocumentDatabaseTestCase { String feedTuning = "<resizing>" + " <initialdocumentcount>1000000000</initialdocumentcount>" + "</resizing>\n"; - verifyInitialDocumentCount(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), - feedTuning, Arrays.asList(INITIAL, INITIAL)); + verifyInitialDocumentCount(List.of(DocType.create("a", "index"), DocType.create("b", "streaming")), + feedTuning, List.of(INITIAL, INITIAL)); } private void assertDocTypeConfig(VespaModel model, String configId, String indexField, String attributeField) { @@ -345,30 +344,30 @@ public class DocumentDatabaseTestCase { @Test void testThatAttributesMaxUnCommittedMemoryIsControlledByFeatureFlag() { - assertAttributesConfigIndependentOfMode("index", Arrays.asList("type1"), - Arrays.asList("test/search/cluster.test/type1"), - ImmutableMap.of("type1", Arrays.asList("f2", "f2_nfa")), + assertAttributesConfigIndependentOfMode("index", List.of("type1"), + List.of("test/search/cluster.test/type1"), + ImmutableMap.of("type1", List.of("f2", "f2_nfa")), new DeployState.Builder().properties(new TestProperties().maxUnCommittedMemory(193452)), 193452); } @Test void testThatAttributesConfigIsProducedForIndexed() { - assertAttributesConfigIndependentOfMode("index", Arrays.asList("type1"), - Arrays.asList("test/search/cluster.test/type1"), - ImmutableMap.of("type1", Arrays.asList("f2", "f2_nfa"))); + assertAttributesConfigIndependentOfMode("index", List.of("type1"), + List.of("test/search/cluster.test/type1"), + ImmutableMap.of("type1", List.of("f2", "f2_nfa"))); } @Test void testThatAttributesConfigIsProducedForStreamingForFastAccessFields() { - assertAttributesConfigIndependentOfMode("streaming", Arrays.asList("type1"), - Arrays.asList("test/search/type1"), - ImmutableMap.of("type1", Arrays.asList("f2"))); + assertAttributesConfigIndependentOfMode("streaming", List.of("type1"), + List.of("test/search/type1"), + ImmutableMap.of("type1", List.of("f2"))); } @Test void testThatAttributesConfigIsNotProducedForStoreOnlyEvenForFastAccessFields() { - assertAttributesConfigIndependentOfMode("store-only", Arrays.asList("type1"), - Arrays.asList("test/search"), Collections.emptyMap()); + assertAttributesConfigIndependentOfMode("store-only", List.of("type1"), + List.of("test/search"), Collections.emptyMap()); } } diff --git a/config-model/src/test/schema-test-files/deployment-with-instances.xml b/config-model/src/test/schema-test-files/deployment-with-instances.xml index 39771ca9d41..f37ff9f6cc6 100644 --- a/config-model/src/test/schema-test-files/deployment-with-instances.xml +++ b/config-model/src/test/schema-test-files/deployment-with-instances.xml @@ -35,7 +35,7 @@ <delay hours='2'/> <parallel> - <instance id="three"> + <instance id="three" tags="a b"> <test/> <staging/> </instance> |