diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2020-04-29 16:07:55 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-04-29 16:07:55 +0200 |
commit | 9499865f8a43aa097841606795a2bea8d0273ef9 (patch) | |
tree | cd3fb502b747ecfea391810227e66f8a19ca9791 | |
parent | ef9877894ca55b8a5bc306a5bcd6837e0b4321a3 (diff) | |
parent | ee27f2cdb3fb79c158f3e5cad2401993f47c1e42 (diff) |
Merge pull request #13109 from vespa-engine/vekterli/add-feature-flag-for-distributor-btree-db
Add feature flag for enabling distributor B-tree bucket DB
6 files changed, 47 insertions, 2 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 4f7cf8575c4..8d4b8549e50 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -88,6 +88,8 @@ public interface ModelContext { // TODO Revisit in May or June 2020 double defaultTopKProbability(); + boolean useDistributorBtreeDb(); + // TODO: Remove once there are no Vespa versions below 7.170 boolean useBucketSpaceMetric(); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index d799af36c3b..475b69c93c2 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -41,6 +41,7 @@ public class TestProperties implements ModelContext.Properties { private boolean useDedicatedNodeForLogserver = false; private boolean useAdaptiveDispatch = false; private double topKProbability = 1.0; + private boolean useDistributorBtreeDb = false; private double defaultTermwiseLimit = 1.0; private double softStartSeconds = 0.0; private double threadPoolSizeFactor = 0.0; @@ -82,6 +83,7 @@ public class TestProperties implements ModelContext.Properties { } @Override public double defaultTopKProbability() { return topKProbability; } + @Override public boolean useDistributorBtreeDb() { return useDistributorBtreeDb; } @Override public boolean useBucketSpaceMetric() { return true; } @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); } @@ -94,6 +96,12 @@ public class TestProperties implements ModelContext.Properties { topKProbability = probability; return this; } + + public TestProperties setUseDistributorBtreeDB(boolean useBtreeDb) { + useDistributorBtreeDb = useBtreeDb; + return this; + } + public TestProperties setSoftStartSeconds(double softStartSeconds) { this.softStartSeconds = softStartSeconds; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 045646cbc5c..42d20b675a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -41,6 +41,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final BucketSplitting bucketSplitting; private final GcOptions gc; private final boolean hasIndexedDocumentType; + private final boolean useBtreeDatabase; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -101,20 +102,24 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl final ModelElement documentsNode = clusterElement.child("documents"); final GcOptions gc = parseGcOptions(documentsNode); final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); + boolean useBtreeDb = deployState.getProperties().useDistributorBtreeDb(); return new DistributorCluster(parent, - new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType); + new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, + hasIndexedDocumentType, useBtreeDb); } } private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, - GcOptions gc, boolean hasIndexedDocumentType) + GcOptions gc, boolean hasIndexedDocumentType, + boolean useBtreeDatabase) { super(parent, "distributor"); this.parent = parent; this.bucketSplitting = bucketSplitting; this.gc = gc; this.hasIndexedDocumentType = hasIndexedDocumentType; + this.useBtreeDatabase = useBtreeDatabase; } @Override @@ -126,6 +131,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl } builder.enable_revert(parent.getPersistence().supportRevert()); builder.disable_bucket_activation(hasIndexedDocumentType == false); + builder.use_btree_database(useBtreeDatabase); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 4d5df7c1965..0e8bd0a41f1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -972,5 +972,20 @@ public class ContentClusterTest extends ContentBaseTest { verifyTopKProbabilityPropertiesControl(0.77); } + private boolean resolveDistributorBtreeDbConfigWithFeatureFlag(boolean flagEnabledBtreeDb) { + VespaModel model = createEnd2EndOneNode(new TestProperties().setUseDistributorBtreeDB(flagEnabledBtreeDb)); + + ContentCluster cc = model.getContentClusters().get("storage"); + var builder = new StorDistributormanagerConfig.Builder(); + cc.getDistributorNodes().getConfig(builder); + + return (new StorDistributormanagerConfig(builder)).use_btree_database(); + } + + @Test + public void default_distributor_btree_usage_controlled_by_properties() { + assertFalse(resolveDistributorBtreeDbConfigWithFeatureFlag(false)); + assertTrue(resolveDistributorBtreeDbConfigWithFeatureFlag(true)); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index cd06f7208e4..08ae4216aaa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -151,6 +151,7 @@ public class ModelContextImpl implements ModelContext { private final boolean isFirstTimeDeployment; private final boolean useAdaptiveDispatch; private final double defaultTopKprobability; + private final boolean useDistributorBtreeDb; private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; private final double defaultSoftStartSeconds; @@ -192,6 +193,8 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); defaultTopKprobability = Flags.DEFAULT_TOP_K_PROBABILITY.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + useDistributorBtreeDb = Flags.USE_DISTRIBUTOR_BTREE_DB.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); threadPoolSizeFactor = Flags.DEFAULT_THREADPOOL_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) @@ -267,6 +270,11 @@ public class ModelContextImpl implements ModelContext { return defaultTopKprobability; } + @Override + public boolean useDistributorBtreeDb() { + return useDistributorBtreeDb; + } + // TODO: Remove @Override public boolean useBucketSpaceMetric() { return true; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 22b5ce419d6..881c3a7dca6 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -166,6 +166,12 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag USE_DISTRIBUTOR_BTREE_DB = defineFeatureFlag( + "use-distributor-btree-db", false, + "Whether to use the new B-tree bucket database in the distributors.", + "Takes effect at restart of distributor process", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag HOST_HARDENING = defineFeatureFlag( "host-hardening", false, "Whether to enable host hardening Linux baseline.", |