diff options
6 files changed, 23 insertions, 5 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 2da2712246f..fd1a7640052 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -84,7 +84,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForBucketMove() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForPruneRemoved() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } - @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default double maxDeadBytesRatio() { return 0.05; } + @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default double maxDeadBytesRatio() { return 0.2; } @ModelFeatureFlag(owners = {"hmusum"}) default int clusterControllerMaxHeapSizeInMb() { return 128; } @ModelFeatureFlag(owners = {"hmusum"}) default int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return 256; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 1d0b95f7a2f..93606236ea7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -52,6 +52,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double feedConcurrency = 0.5; private boolean useBucketExecutorForPruneRemoved; private boolean enableFeedBlockInDistributor = true; + private double maxDeadBytesRatio = 0.2; private int clusterControllerMaxHeapSizeInMb = 256; private int metricsProxyMaxHeapSizeInMb = 256; private int maxActivationInhibitedOutOfSyncGroups = 0; @@ -91,6 +92,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } + @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @@ -201,6 +203,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties maxDeadBytesRatio(double ratio) { + maxDeadBytesRatio = ratio; + return this; + } + public TestProperties clusterControllerMaxHeapSizeInMb(int heapSize) { clusterControllerMaxHeapSizeInMb = heapSize; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 18580249ddc..ec5b5c03402 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -66,6 +66,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final double defaultFeedConcurrency; private final boolean useBucketExecutorForPruneRemoved; + private final double defaultMaxDeadBytesRatio; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -211,6 +212,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); useBucketExecutorForPruneRemoved = featureFlags.useBucketExecutorForPruneRemoved(); + defaultMaxDeadBytesRatio = featureFlags.maxDeadBytesRatio(); } public void setVisibilityDelay(double delay) { @@ -377,6 +379,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> .configid(getConfigId()) .visibilitydelay(visibilityDelay) .global(globalDocType); + ddbB.allocation.max_dead_bytes_ratio(defaultMaxDeadBytesRatio); if (hasIndexingModeStreaming(type)) { hasAnyNonIndexedCluster = true; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index d479f5ff7a9..58d25cd7209 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -170,6 +170,7 @@ public class ModelContextImpl implements ModelContext { private final double feedConcurrency; private final boolean useBucketExecutorForPruneRemoved; private final boolean enableFeedBlockInDistributor; + private final double maxDeadBytesRatio; private final int clusterControllerMaxHeapSizeInMb; private final ToIntFunction<ClusterSpec.Type> metricsProxyMaxHeapSizeInMb; private final List<String> allowedAthenzProxyIdentities; @@ -194,6 +195,7 @@ public class ModelContextImpl implements ModelContext { this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); this.useBucketExecutorForPruneRemoved = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_PRUNE_REMOVED); this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); + this.maxDeadBytesRatio = flagValue(source, appId, Flags.MAX_DEAD_BYTES_RATIO); this.clusterControllerMaxHeapSizeInMb = flagValue(source, appId, Flags.CLUSTER_CONTROLLER_MAX_HEAP_SIZE_IN_MB); this.metricsProxyMaxHeapSizeInMb = type -> Flags.METRICS_PROXY_MAX_HEAP_SIZE_IN_MB.bindTo(source).with(CLUSTER_TYPE, type.name()).value(); this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); @@ -218,6 +220,7 @@ public class ModelContextImpl implements ModelContext { @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } + @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb.applyAsInt(type); } @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index f7b8434ca8e..a9ac1c9cefc 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -145,7 +145,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_PRUNE_REMOVED = defineFeatureFlag( - "use-bucket-executor-for-prune-removed", false, + "use-bucket-executor-for-prune-removed", true, List.of("baldersheim"), "2021-05-04", "2021-06-01", "Wheter to use content-level bucket executor or legacy frozen buckets for prune removed", "Takes effect on next internal redeployment", @@ -165,6 +165,13 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag MAX_DEAD_BYTES_RATIO = defineDoubleFlag( + "max-dead-bytes-ratio", 0.05, + List.of("baldersheim", "geirst","toregge"), "2021-02-03", "2021-06-01", + "max ratio of dead to used memory bytes in large data structures before compaction is attempted", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag SYNC_HOST_LOGS_TO_S3_BUCKET = defineStringFlag( "sync-host-logs-to-s3-bucket", "", List.of("andreer", "valerijf"), "2021-02-10", "2021-05-01", "Host-admin should sync host logs to an S3 bucket named by this flag. If left empty, sync is disabled", diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index eb4d8b4049a..835ac5ae286 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -353,7 +353,7 @@ pruneremoveddocumentsinterval double default=0.0 pruneremoveddocumentsage double default=1209600.0 ## Set to true to enable bucket locking via content layer -pruneremoveddocuments.usebucketexecutor bool default=false +pruneremoveddocuments.usebucketexecutor bool default=true ## Minimum size of packets to compress (0 means no compression) ## @@ -416,14 +416,12 @@ lidspacecompaction.removebatchblockrate double default=0.5 lidspacecompaction.removeblockrate double default=100.0 ## Set to true to enable bucket locking via content layer -## DEPRECATED and unused lidspacecompaction.usebucketexecutor bool default=true ## Maximum docs to move in single operation per bucket bucketmove.maxdocstomoveperbucket int default=1 ## Set to true to enable bucket locking via content layer -## DEPRECATED and unused bucketmove.usebucketexecutor bool default=true ## This is the maximum value visibilitydelay you can have. |