diff options
10 files changed, 32 insertions, 77 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 99bcd59d791..d47e46b9459 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -89,8 +89,6 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedNiceness() { return 0.0; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } - @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { return 16; } - @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { return 100; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean sharedStringRepoNoReclaim() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean loadCodeAsHugePages() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @@ -132,6 +130,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}, removeAfter="7.last") default int persistenceThrottlingWindowSize() { return -1; } @ModelFeatureFlag(owners = {"vekterli"}, removeAfter="7.last") default double persistenceThrottlingWsResizeRate() { return 3; } @ModelFeatureFlag(owners = {"vekterli"}, removeAfter="7.last") default boolean persistenceThrottlingOfMergeFeedOps() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default int maxConcurrentMergesPerNode() { return 16; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default int maxMergeQueueSize() { return 100; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 2520dda2ed0..0c7785aabd4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -55,8 +55,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; - private int maxConcurrentMergesPerNode = 16; - private int maxMergeQueueSize = 100; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = Collections.emptyList(); private double resourceLimitDisk = 0.75; @@ -111,8 +109,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } - @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } - @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @@ -206,15 +202,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setMaxConcurrentMergesPerNode(int maxConcurrentMergesPerNode) { - this.maxConcurrentMergesPerNode = maxConcurrentMergesPerNode; - return this; - } - public TestProperties setMaxMergeQueueSize(int maxMergeQueueSize) { - this.maxMergeQueueSize = maxMergeQueueSize; - return this; - } - public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 4c74282c061..06453bffaaf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 86785ea20d2..4298488b1fd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content.storagecluster; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -11,10 +10,10 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster; */ public class StorServerProducer implements StorServerConfig.Producer { public static class Builder { - StorServerProducer build(ModelContext.Properties properties, ModelElement element) { + StorServerProducer build(ModelElement element) { ModelElement tuning = element.child("tuning"); - StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element), properties.featureFlags()); + StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element)); if (tuning == null) return producer; ModelElement merges = tuning.child("merges"); @@ -43,10 +42,8 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } - StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { + StorServerProducer(String clusterName) { this.clusterName = clusterName; - maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); - queueSize = featureFlags.maxMergeQueueSize(); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index da82a69842a..9b59f6db742 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -38,7 +38,7 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> ContentCluster.getClusterId(clusterElem), new FileStorProducer.Builder().build(deployState.getProperties(), cluster, clusterElem), new IntegrityCheckerProducer.Builder().build(cluster, clusterElem), - new StorServerProducer.Builder().build(deployState.getProperties(), clusterElem), + new StorServerProducer.Builder().build(clusterElem), new StorVisitorProducer.Builder().build(clusterElem), new PersistenceProducer.Builder().build(clusterElem)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 48cee3fd397..f3a59733ece 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -120,9 +120,7 @@ public class StorageClusterTest { parse(cluster("foofighters", joinLines( "<tuning>", " <merges max-per-node=\"1K\" max-queue-size=\"10K\"/>", - "</tuning>")), - new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37) - ).getConfig(builder); + "</tuning>"))).getConfig(builder); StorServerConfig config = new StorServerConfig(builder); assertEquals(1024, config.max_merges_per_node()); @@ -174,9 +172,9 @@ public class StorageClusterTest { @Test void testMergeFeatureFlags() { - var config = configFromProperties(new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37)); - assertEquals(37, config.max_merges_per_node()); - assertEquals(1919, config.max_merge_queue_size()); + var config = configFromProperties(new TestProperties()); + assertEquals(16, config.max_merges_per_node()); + assertEquals(100, config.max_merge_queue_size()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 8830e5484b3..ee885cdf43e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.search.IndexedSearchCluster; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -55,7 +54,7 @@ public class DocumentDatabaseTestCase { @Test void requireThatMixedModeConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 1.0); + verifyConcurrency(List.of(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 1.0); } @Test @@ -63,7 +62,7 @@ public class DocumentDatabaseTestCase { String feedTuning = "<feeding>" + " <concurrency>0.7</concurrency>" + "</feeding>\n"; - verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), feedTuning, 0.7); + verifyConcurrency(List.of(DocType.create("a", "index"), DocType.create("b", "streaming")), feedTuning, 0.7); } @Test @@ -77,11 +76,11 @@ public class DocumentDatabaseTestCase { } private void verifyConcurrency(String mode, String xmlTuning, double expectedConcurrency, double featureFlagConcurrency) { - verifyConcurrency(Arrays.asList(DocType.create("a", mode)), xmlTuning, expectedConcurrency, featureFlagConcurrency); + verifyConcurrency(List.of(DocType.create("a", mode)), xmlTuning, expectedConcurrency, featureFlagConcurrency); } private void verifyConcurrency(String mode, String xmlTuning, double expectedConcurrency) { - verifyConcurrency(Arrays.asList(DocType.create("a", mode)), xmlTuning, expectedConcurrency, null); + verifyConcurrency(List.of(DocType.create("a", mode)), xmlTuning, expectedConcurrency, null); } private void verifyConcurrency(List<DocType> nameAndModes, String xmlTuning, double expectedConcurrency) { @@ -114,14 +113,14 @@ public class DocumentDatabaseTestCase { @Test void requireFeedNicenessIsReflected() { - verifyFeedNiceness(Arrays.asList(DocType.create("a", "index")), 0.0, null); - verifyFeedNiceness(Arrays.asList(DocType.create("a", "index")), 0.32, 0.32); + verifyFeedNiceness(List.of(DocType.create("a", "index")), 0.0, null); + verifyFeedNiceness(List.of(DocType.create("a", "index")), 0.32, 0.32); } @Test void requireThatModeIsSet() { var tester = new SchemaTester(); - VespaModel model = tester.createModel(Arrays.asList(DocType.create("a", "index"), + VespaModel model = tester.createModel(List.of(DocType.create("a", "index"), DocType.create("b", "streaming"), DocType.create("c", "store-only")), ""); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); @@ -150,8 +149,8 @@ public class DocumentDatabaseTestCase { @Test void requireThatMixedModeInitialDocumentCountIsReflectedCorrectlyForDefault() { final long DEFAULT = 1024L; - verifyInitialDocumentCount(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), - "", Arrays.asList(DEFAULT, DEFAULT)); + verifyInitialDocumentCount(List.of(DocType.create("a", "index"), DocType.create("b", "streaming")), + "", List.of(DEFAULT, DEFAULT)); } @Test @@ -160,8 +159,8 @@ public class DocumentDatabaseTestCase { String feedTuning = "<resizing>" + " <initialdocumentcount>1000000000</initialdocumentcount>" + "</resizing>\n"; - verifyInitialDocumentCount(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), - feedTuning, Arrays.asList(INITIAL, INITIAL)); + verifyInitialDocumentCount(List.of(DocType.create("a", "index"), DocType.create("b", "streaming")), + feedTuning, List.of(INITIAL, INITIAL)); } private void assertDocTypeConfig(VespaModel model, String configId, String indexField, String attributeField) { @@ -345,30 +344,30 @@ public class DocumentDatabaseTestCase { @Test void testThatAttributesMaxUnCommittedMemoryIsControlledByFeatureFlag() { - assertAttributesConfigIndependentOfMode("index", Arrays.asList("type1"), - Arrays.asList("test/search/cluster.test/type1"), - ImmutableMap.of("type1", Arrays.asList("f2", "f2_nfa")), + assertAttributesConfigIndependentOfMode("index", List.of("type1"), + List.of("test/search/cluster.test/type1"), + ImmutableMap.of("type1", List.of("f2", "f2_nfa")), new DeployState.Builder().properties(new TestProperties().maxUnCommittedMemory(193452)), 193452); } @Test void testThatAttributesConfigIsProducedForIndexed() { - assertAttributesConfigIndependentOfMode("index", Arrays.asList("type1"), - Arrays.asList("test/search/cluster.test/type1"), - ImmutableMap.of("type1", Arrays.asList("f2", "f2_nfa"))); + assertAttributesConfigIndependentOfMode("index", List.of("type1"), + List.of("test/search/cluster.test/type1"), + ImmutableMap.of("type1", List.of("f2", "f2_nfa"))); } @Test void testThatAttributesConfigIsProducedForStreamingForFastAccessFields() { - assertAttributesConfigIndependentOfMode("streaming", Arrays.asList("type1"), - Arrays.asList("test/search/type1"), - ImmutableMap.of("type1", Arrays.asList("f2"))); + assertAttributesConfigIndependentOfMode("streaming", List.of("type1"), + List.of("test/search/type1"), + ImmutableMap.of("type1", List.of("f2"))); } @Test void testThatAttributesConfigIsNotProducedForStoreOnlyEvenForFastAccessFields() { - assertAttributesConfigIndependentOfMode("store-only", Arrays.asList("type1"), - Arrays.asList("test/search"), Collections.emptyMap()); + assertAttributesConfigIndependentOfMode("store-only", List.of("type1"), + List.of("test/search"), Collections.emptyMap()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index f9d8c37ea9d..e6c3eaabdfd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -180,8 +180,6 @@ public class ModelContextImpl implements ModelContext { private final List<String> allowedAthenzProxyIdentities; private final int maxActivationInhibitedOutOfSyncGroups; private final ToIntFunction<ClusterSpec.Type> jvmOmitStackTraceInFastThrow; - private final int maxConcurrentMergesPerContentNode; - private final int maxMergeQueueSize; private final double resourceLimitDisk; private final double resourceLimitMemory; private final double minNodeRatioPerGroup; @@ -225,8 +223,6 @@ public class ModelContextImpl implements ModelContext { this.allowedAthenzProxyIdentities = flagValue(source, appId, version, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, version, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, version, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); - this.maxConcurrentMergesPerContentNode = flagValue(source, appId, version, Flags.MAX_CONCURRENT_MERGES_PER_NODE); - this.maxMergeQueueSize = flagValue(source, appId, version, Flags.MAX_MERGE_QUEUE_SIZE); this.resourceLimitDisk = flagValue(source, appId, version, PermanentFlags.RESOURCE_LIMIT_DISK); this.resourceLimitMemory = flagValue(source, appId, version, PermanentFlags.RESOURCE_LIMIT_MEMORY); this.minNodeRatioPerGroup = flagValue(source, appId, version, Flags.MIN_NODE_RATIO_PER_GROUP); @@ -273,8 +269,6 @@ public class ModelContextImpl implements ModelContext { @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return translateJvmOmitStackTraceInFastThrowIntToString(jvmOmitStackTraceInFastThrow, type); } - @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; } - @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 69796e4d0f8..71df5a8829e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -17,7 +17,6 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.FileDistribution; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.Quota; import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.AllocatedHosts; @@ -35,13 +34,11 @@ import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; -import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory; import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.modelfactory.AllocatedHostsFromAllModels; -import com.yahoo.vespa.config.server.modelfactory.LegacyFlags; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; @@ -73,8 +70,6 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.zip.ZipException; -import static com.yahoo.vespa.config.server.ConfigServerSpec.fromConfig; - /** * A SessionPreparer is responsible for preparing a session given an application package. * diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 24e9ea5ef7c..5766b059d98 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -233,20 +233,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( - "max-concurrent-merges-per-node", 16, - List.of("balder", "vekterli"), "2021-06-06", "2022-12-01", - "Specifies max concurrent merges per content node.", - "Takes effect at redeploy", - ZONE_ID, APPLICATION_ID); - - public static final UnboundIntFlag MAX_MERGE_QUEUE_SIZE = defineIntFlag( - "max-merge-queue-size", 100, - List.of("balder", "vekterli"), "2021-06-06", "2022-12-01", - "Specifies max size of merge queue.", - "Takes effect at redeploy", - ZONE_ID, APPLICATION_ID); - public static final UnboundDoubleFlag MIN_NODE_RATIO_PER_GROUP = defineDoubleFlag( "min-node-ratio-per-group", 0.0, List.of("geirst", "vekterli"), "2021-07-16", "2022-11-01", |