From 5e425b8215b05e573ee9321f1cae509bb259d370 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Wed, 12 Jan 2022 14:10:53 +0100 Subject: Add feature flag for controlling throttling of async persistence operations Currently defaults to `UNLIMITED` throttling policy, i.e. legacy behavior. --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 1 + 1 file changed, 1 insertion(+) (limited to 'config-model-api') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 2ea1ef186d2..42718658fbf 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -114,6 +114,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default double tlsSizeFraction() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } + @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ -- cgit v1.2.3 From f9e15817be23063f91c6bff21b61cf63db2279c9 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 12 Jan 2022 16:28:14 +0100 Subject: No longer need for featureflag control of docstore compression. --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 4 ++-- .../main/java/com/yahoo/config/model/deploy/TestProperties.java | 7 ------- .../java/com/yahoo/vespa/model/content/ContentSearchCluster.java | 8 ++++---- .../com/yahoo/vespa/model/content/ContentSchemaClusterTest.java | 9 ++------- .../com/yahoo/vespa/config/server/deploy/ModelContextImpl.java | 3 --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 7 ------- 6 files changed, 8 insertions(+), 30 deletions(-) (limited to 'config-model-api') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 42718658fbf..8d587577d73 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -83,7 +83,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int metricsproxyNumThreads() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.526") default int largeRankExpressionLimit() { return 8192; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int largeRankExpressionLimit() { return 8192; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { throw new UnsupportedOperationException("TODO specify default value"); } @@ -91,7 +91,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double diskBloatFactor() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default int docstoreCompressionLevel() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int docstoreCompressionLevel() { return 3; } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 2dc91ab1b1f..011d525b0bf 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -67,7 +67,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean containerDumpHeapOnShutdownTimeout = false; private double containerShutdownTimeout = 50.0; private int distributorMergeBusyWait = 10; - private int docstoreCompressionLevel = 9; private int maxUnCommittedMemory = 123456; private double diskBloatFactor = 0.25; private boolean distributorEnhancedMaintenanceScheduling = false; @@ -126,7 +125,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } @Override public double diskBloatFactor() { return diskBloatFactor; } - @Override public int docstoreCompressionLevel() { return docstoreCompressionLevel; } @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @@ -142,11 +140,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties docstoreCompressionLevel(int docstoreCompressionLevel) { - this.docstoreCompressionLevel = docstoreCompressionLevel; - return this; - } - public TestProperties diskBloatFactor(double diskBloatFactor) { this.diskBloatFactor = diskBloatFactor; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 54d09bacfa9..f7ba2fb2fe3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -43,6 +43,8 @@ import static java.util.stream.Collectors.toList; */ public class ContentSearchCluster extends AbstractConfigProducer implements ProtonConfig.Producer, DispatchConfig.Producer { + private static final int DEFAULT_DOC_STORE_COMPRESSION_LEVEL = 3; + private final boolean flushOnShutdown; private final Boolean syncTransactionLog; @@ -69,7 +71,6 @@ public class ContentSearchCluster extends AbstractConfigProducer private final ProtonConfig.Feeding.Shared_field_writer_executor.Enum sharedFieldWriterExecutor; private final double defaultFeedConcurrency; private final double defaultDiskBloatFactor; - private final int defaultDocStoreCompressionLevel; private final boolean forwardIssuesToQrs; private final int defaultMaxCompactBuffers; @@ -224,7 +225,6 @@ public class ContentSearchCluster extends AbstractConfigProducer this.sharedFieldWriterExecutor = convertSharedFieldWriterExecutor(featureFlags.sharedFieldWriterExecutor()); this.defaultFeedConcurrency = featureFlags.feedConcurrency(); this.defaultDiskBloatFactor = featureFlags.diskBloatFactor(); - this.defaultDocStoreCompressionLevel = featureFlags.docstoreCompressionLevel(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); } @@ -425,8 +425,8 @@ public class ContentSearchCluster extends AbstractConfigProducer } builder.flush.memory.diskbloatfactor(defaultDiskBloatFactor); builder.flush.memory.each.diskbloatfactor(defaultDiskBloatFactor); - builder.summary.log.chunk.compression.level(defaultDocStoreCompressionLevel); - builder.summary.log.compact.compression.level(defaultDocStoreCompressionLevel); + builder.summary.log.chunk.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL); + builder.summary.log.compact.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL); builder.forward_issues(forwardIssuesToQrs); int numDocumentDbs = builder.documentdb.size(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java index 51badae2746..97e097fbe24 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java @@ -33,7 +33,7 @@ import static org.junit.Assert.assertTrue; */ public class ContentSchemaClusterTest { - private static double EPSILON = 0.000001; + private static final double EPSILON = 0.000001; private static ContentCluster createClusterWithOneDocumentType() throws Exception { return createCluster(new ContentClusterBuilder().getXml()); @@ -261,13 +261,8 @@ public class ContentSchemaClusterTest { } @Test - public void verifyControlOfDocStoreCompression() throws Exception { + public void verifyDefaultDocStoreCompression() throws Exception { ProtonConfig cfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml())); - assertEquals(9, cfg.summary().log().chunk().compression().level()); - assertEquals(9, cfg.summary().log().compact().compression().level()); - - cfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml(), - new DeployState.Builder().properties(new TestProperties().docstoreCompressionLevel(3)))); assertEquals(3, cfg.summary().log().chunk().compression().level()); assertEquals(3, cfg.summary().log().compact().compression().level()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index b32c82399f6..8221206eaf5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -191,7 +191,6 @@ public class ModelContextImpl implements ModelContext { private final boolean containerDumpHeapOnShutdownTimeout; private final double containerShutdownTimeout; private final int distributorMergeBusyWait; - private final int docstoreCompressionLevel; private final double diskBloatFactor; private final boolean distributorEnhancedMaintenanceScheduling; private final int maxUnCommittedMemory; @@ -236,7 +235,6 @@ public class ModelContextImpl implements ModelContext { this.containerDumpHeapOnShutdownTimeout = flagValue(source, appId, Flags.CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT); this.containerShutdownTimeout = flagValue(source, appId,Flags.CONTAINER_SHUTDOWN_TIMEOUT); this.distributorMergeBusyWait = flagValue(source, appId, Flags.DISTRIBUTOR_MERGE_BUSY_WAIT); - this.docstoreCompressionLevel = flagValue(source, appId, Flags.DOCSTORE_COMPRESSION_LEVEL); this.diskBloatFactor = flagValue(source, appId, Flags.DISK_BLOAT_FACTOR); this.distributorEnhancedMaintenanceScheduling = flagValue(source, appId, Flags.DISTRIBUTOR_ENHANCED_MAINTENANCE_SCHEDULING); this.maxUnCommittedMemory = flagValue(source, appId, Flags.MAX_UNCOMMITTED_MEMORY);; @@ -284,7 +282,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } @Override public double diskBloatFactor() { return diskBloatFactor; } - @Override public int docstoreCompressionLevel() { return docstoreCompressionLevel; } @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean forwardIssuesAsErrors() { return forwardIssuesAsErrors; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 5dc418e6e6c..bb291ad48fa 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -166,13 +166,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundIntFlag DOCSTORE_COMPRESSION_LEVEL = defineIntFlag( - "docstore-compression-level", 3, - List.of("baldersheim"), "2021-10-08", "2022-02-01", - "Default compression level used for document store", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( "enable-feed-block-in-distributor", true, List.of("geirst"), "2021-01-27", "2022-01-31", -- cgit v1.2.3 From ed4938a382461c93062482fc29e76b472688165f Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 12 Jan 2022 16:47:31 +0100 Subject: GC featureflag control of max size of transaction log --- .../java/com/yahoo/config/model/api/ModelContext.java | 2 +- .../com/yahoo/config/model/deploy/TestProperties.java | 7 ------- .../yahoo/vespa/model/content/ContentClusterTest.java | 16 +++++----------- .../vespa/config/server/deploy/ModelContextImpl.java | 3 --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 7 ------- 5 files changed, 6 insertions(+), 29 deletions(-) (limited to 'config-model-api') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 42718658fbf..92868c1ae8e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -111,7 +111,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej", "baldersheim"}) default boolean useV8DocManagerCfg() { return false; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } - @ModelFeatureFlag(owners = {"baldersheim"}) default double tlsSizeFraction() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double tlsSizeFraction() { return 0.02; } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 2dc91ab1b1f..4260ceb7ec5 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -76,7 +76,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private List zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; - private double tlsSizeFraction = 0.02; private String persistenceAsyncThrottling = "UNLIMITED"; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @@ -134,7 +133,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } - @Override public double tlsSizeFraction() { return tlsSizeFraction; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { @@ -353,11 +351,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties tlsSizeFraction(double tlsSizeFraction) { - this.tlsSizeFraction = tlsSizeFraction; - return this; - } - public TestProperties setPersistenceAsyncThrottling(String type) { this.persistenceAsyncThrottling = type; return this; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 4295f0aa6ec..0c1c2c74dd1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -45,7 +45,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalDouble; import java.util.OptionalInt; import static org.junit.Assert.assertEquals; @@ -1046,11 +1045,9 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); } - private long resolveMaxTLSSize(OptionalDouble tlsSizeFraction, Optional flavor) throws Exception { + private long resolveMaxTLSSize(Optional flavor) throws Exception { TestProperties testProperties = new TestProperties(); - if (tlsSizeFraction.isPresent()) { - testProperties.tlsSizeFraction(tlsSizeFraction.getAsDouble()); - } + ContentCluster cc = createOneNodeCluster(testProperties, flavor); ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); cc.getSearch().getSearchNodes().get(0).getConfig(protonBuilder); @@ -1058,13 +1055,10 @@ public class ContentClusterTest extends ContentBaseTest { return protonConfig.flush().memory().maxtlssize(); } @Test - public void default_max_tls_size_controlled_by_properties() throws Exception { + public void verifyt_max_tls_size() throws Exception { var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100))); - assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.empty())); - assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.empty())); - assertEquals(2147483648L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.of(flavor))); - assertEquals(2147483648L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.of(flavor))); - assertEquals(3221225472L, resolveMaxTLSSize(OptionalDouble.of(0.03), Optional.of(flavor))); + assertEquals(21474836480L, resolveMaxTLSSize(Optional.empty())); + assertEquals(2147483648L, resolveMaxTLSSize(Optional.of(flavor))); } void assertZookeeperServerImplementation(String expectedClassName, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index b32c82399f6..584d316546d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -203,7 +203,6 @@ public class ModelContextImpl implements ModelContext { private final boolean useV8DocManagerCfg; private final int maxCompactBuffers; private final boolean failDeploymentWithInvalidJvmOptions; - private final double tlsSizeFraction; private final List ignoredHttpUserAgents; private final boolean enableServerOcspStapling; private final String persistenceAsyncThrottling; @@ -248,7 +247,6 @@ public class ModelContextImpl implements ModelContext { this.useV8DocManagerCfg = flagValue(source, appId, Flags.USE_V8_DOC_MANAGER_CFG); this.maxCompactBuffers = flagValue(source, appId, Flags.MAX_COMPACT_BUFFERS); this.failDeploymentWithInvalidJvmOptions = flagValue(source, appId, Flags.FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS); - this.tlsSizeFraction = flagValue(source, appId, Flags.TLS_SIZE_FRACTION); this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); @@ -295,7 +293,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useV8DocManagerCfg() { return useV8DocManagerCfg; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } - @Override public double tlsSizeFraction() { return tlsSizeFraction; } @Override public List ignoredHttpUserAgents() { return ignoredHttpUserAgents; } @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 5dc418e6e6c..4b9eff73a22 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -52,13 +52,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundDoubleFlag TLS_SIZE_FRACTION = defineDoubleFlag( - "tls-size-fraction", 0.02, - List.of("baldersheim"), "2021-12-20", "2022-02-01", - "Fraction of disk available for transaction log", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( "feed-sequencer-type", "THROUGHPUT", List.of("baldersheim"), "2020-12-02", "2022-02-01", -- cgit v1.2.3 From 0c21f034b1e0c8492eb9713b49f2c1081eb5b8c6 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 12 Jan 2022 16:32:17 +0100 Subject: GC newLocationBrokerLogic and requireConnectivityCheck that rolled out autumn 2021. --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 2 -- 1 file changed, 2 deletions(-) (limited to 'config-model-api') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 42718658fbf..b5293987553 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -96,11 +96,9 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } @ModelFeatureFlag(owners = {"hmusum"}) default String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return ""; } - @ModelFeatureFlag(owners = {"arnej"}) default boolean requireConnectivityCheck() { return true; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.8; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default double minNodeRatioPerGroup() { return 0.0; } - @ModelFeatureFlag(owners = {"arnej"}) default boolean newLocationBrokerLogic() { return true; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int distributorMergeBusyWait() { return 10; } @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean distributorEnhancedMaintenanceScheduling() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean forwardIssuesAsErrors() { return true; } -- cgit v1.2.3 From ef469cc08d7206dfb72b5cab842053264633a1cc Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 12 Jan 2022 17:04:19 +0100 Subject: GC featureflag control of disk bloat factor --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 2 +- .../main/java/com/yahoo/config/model/deploy/TestProperties.java | 7 ------- .../java/com/yahoo/vespa/model/content/ContentSearchCluster.java | 7 +++---- .../com/yahoo/vespa/model/content/ContentSchemaClusterTest.java | 8 +------- .../com/yahoo/vespa/config/server/deploy/ModelContextImpl.java | 3 --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 7 ------- 6 files changed, 5 insertions(+), 29 deletions(-) (limited to 'config-model-api') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 8d587577d73..0656d718ccc 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -90,7 +90,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean ignoreMergeQueueLimit() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default double diskBloatFactor() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double diskBloatFactor() { return 0.25; } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int docstoreCompressionLevel() { return 3; } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List allowedAthenzProxyIdentities() { return List.of(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 011d525b0bf..8a6b7e07377 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -68,7 +68,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double containerShutdownTimeout = 50.0; private int distributorMergeBusyWait = 10; private int maxUnCommittedMemory = 123456; - private double diskBloatFactor = 0.25; private boolean distributorEnhancedMaintenanceScheduling = false; private boolean asyncApplyBucketDiff = false; private boolean unorderedMergeChaining = false; @@ -124,7 +123,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } - @Override public double diskBloatFactor() { return diskBloatFactor; } @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @@ -140,11 +138,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties diskBloatFactor(double diskBloatFactor) { - this.diskBloatFactor = diskBloatFactor; - return this; - } - public TestProperties containerDumpHeapOnShutdownTimeout(boolean value) { containerDumpHeapOnShutdownTimeout = value; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index f7ba2fb2fe3..cadba086e36 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -44,6 +44,7 @@ import static java.util.stream.Collectors.toList; public class ContentSearchCluster extends AbstractConfigProducer implements ProtonConfig.Producer, DispatchConfig.Producer { private static final int DEFAULT_DOC_STORE_COMPRESSION_LEVEL = 3; + private static final double DEFAULT_DISK_BLOAT = 0.25; private final boolean flushOnShutdown; private final Boolean syncTransactionLog; @@ -70,7 +71,6 @@ public class ContentSearchCluster extends AbstractConfigProducer private final int feedMasterTaskLimit; private final ProtonConfig.Feeding.Shared_field_writer_executor.Enum sharedFieldWriterExecutor; private final double defaultFeedConcurrency; - private final double defaultDiskBloatFactor; private final boolean forwardIssuesToQrs; private final int defaultMaxCompactBuffers; @@ -224,7 +224,6 @@ public class ContentSearchCluster extends AbstractConfigProducer this.feedMasterTaskLimit = featureFlags.feedMasterTaskLimit(); this.sharedFieldWriterExecutor = convertSharedFieldWriterExecutor(featureFlags.sharedFieldWriterExecutor()); this.defaultFeedConcurrency = featureFlags.feedConcurrency(); - this.defaultDiskBloatFactor = featureFlags.diskBloatFactor(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); } @@ -423,8 +422,8 @@ public class ContentSearchCluster extends AbstractConfigProducer } else { builder.feeding.concurrency(defaultFeedConcurrency); } - builder.flush.memory.diskbloatfactor(defaultDiskBloatFactor); - builder.flush.memory.each.diskbloatfactor(defaultDiskBloatFactor); + builder.flush.memory.diskbloatfactor(DEFAULT_DISK_BLOAT); + builder.flush.memory.each.diskbloatfactor(DEFAULT_DISK_BLOAT); builder.summary.log.chunk.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL); builder.summary.log.compact.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL); builder.forward_issues(forwardIssuesToQrs); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java index 97e097fbe24..68e722f45d3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java @@ -268,15 +268,9 @@ public class ContentSchemaClusterTest { } @Test - public void verifyControlOfDiskBloatFactor() throws Exception { + public void verifyDefaultDiskBloatFactor() throws Exception { var defaultCfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml())); assertEquals(0.25, defaultCfg.flush().memory().diskbloatfactor(), EPSILON); assertEquals(0.25, defaultCfg.flush().memory().each().diskbloatfactor(), EPSILON); - - var controlledCfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml(), - new DeployState.Builder().properties(new TestProperties().diskBloatFactor(0.31)) - )); - assertEquals(0.31, controlledCfg.flush().memory().diskbloatfactor(), EPSILON); - assertEquals(0.31, controlledCfg.flush().memory().each().diskbloatfactor(), EPSILON); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 8221206eaf5..85110118f53 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -191,7 +191,6 @@ public class ModelContextImpl implements ModelContext { private final boolean containerDumpHeapOnShutdownTimeout; private final double containerShutdownTimeout; private final int distributorMergeBusyWait; - private final double diskBloatFactor; private final boolean distributorEnhancedMaintenanceScheduling; private final int maxUnCommittedMemory; private final boolean forwardIssuesAsErrors; @@ -235,7 +234,6 @@ public class ModelContextImpl implements ModelContext { this.containerDumpHeapOnShutdownTimeout = flagValue(source, appId, Flags.CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT); this.containerShutdownTimeout = flagValue(source, appId,Flags.CONTAINER_SHUTDOWN_TIMEOUT); this.distributorMergeBusyWait = flagValue(source, appId, Flags.DISTRIBUTOR_MERGE_BUSY_WAIT); - this.diskBloatFactor = flagValue(source, appId, Flags.DISK_BLOAT_FACTOR); this.distributorEnhancedMaintenanceScheduling = flagValue(source, appId, Flags.DISTRIBUTOR_ENHANCED_MAINTENANCE_SCHEDULING); this.maxUnCommittedMemory = flagValue(source, appId, Flags.MAX_UNCOMMITTED_MEMORY);; this.forwardIssuesAsErrors = flagValue(source, appId, PermanentFlags.FORWARD_ISSUES_AS_ERRORS); @@ -281,7 +279,6 @@ public class ModelContextImpl implements ModelContext { @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } - @Override public double diskBloatFactor() { return diskBloatFactor; } @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean forwardIssuesAsErrors() { return forwardIssuesAsErrors; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index bb291ad48fa..23edf6b77d7 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -159,13 +159,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundDoubleFlag DISK_BLOAT_FACTOR = defineDoubleFlag( - "disk-bloat-factor", 0.25, - List.of("baldersheim"), "2021-10-08", "2022-02-01", - "Amount of bloat allowed before compacting file", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( "enable-feed-block-in-distributor", true, List.of("geirst"), "2021-01-27", "2022-01-31", -- cgit v1.2.3