diff options
author | Geir Storli <geirstorli@yahoo.no> | 2018-02-28 09:18:09 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-02-28 09:18:09 +0100 |
commit | a35db3bc1005adb1d49335acef98aba1761368b9 (patch) | |
tree | fa136a626fd34b981f7fc34114a38531866f8c29 | |
parent | c2d3acc84c102d9cc91ea8d71d607b16220ab58d (diff) | |
parent | 52655b4add1d1771b3434e005628bdc0ef633bf6 (diff) |
Merge pull request #5166 from vespa-engine/geirst/add-switch-to-enable-multiple-bucket-spaces-in-cluster-controller
Add switch to enable multiple bucket spaces in cluster controller.
8 files changed, 38 insertions, 8 deletions
diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java index 8b80b0f7be5..2a9c076a060 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java @@ -73,6 +73,7 @@ public class ClusterControllerClusterConfigurer { options.distributionBits = config.ideal_distribution_bits(); options.minNodeRatioPerGroup = config.min_node_ratio_per_group(); options.setMaxDeferredTaskVersionWaitTime(Duration.ofMillis((int)(config.max_deferred_task_version_wait_time_sec() * 1000))); + options.enableMultipleBucketSpaces = config.enable_multiple_bucket_spaces(); } private void configure(SlobroksConfig config) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java index 4a2ce1420de..3b8cb14e977 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java @@ -56,7 +56,7 @@ public class ClusterStatsAggregator { if (!hasUpdatesFromAllDistributors()) { return true; } - AggregatedStatsMergePendingChecker checker = new AggregatedStatsMergePendingChecker(aggregatedStats); + MergePendingChecker checker = createMergePendingChecker(); for (Iterator<ContentNodeStats> itr = aggregatedStats.iterator(); itr.hasNext(); ) { ContentNodeStats stats = itr.next(); if (checker.mayHaveMergesPending(FixedBucketSpaces.globalSpace(), stats.getNodeIndex())) { @@ -66,6 +66,10 @@ public class ClusterStatsAggregator { return false; } + MergePendingChecker createMergePendingChecker() { + return new AggregatedStatsMergePendingChecker(aggregatedStats); + } + /** * Update the aggregator with the newest available stats from a distributor. */ diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsChangeTracker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsChangeTracker.java index 3ed6fc332ed..ed32a5f4b45 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsChangeTracker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsChangeTracker.java @@ -35,4 +35,5 @@ public class ClusterStatsChangeTracker { } return false; } + } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index e100fea780a..e755c2b8117 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -824,7 +824,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } private ClusterStateDeriver createBucketSpaceStateDeriver() { - return new MaintenanceWhenPendingGlobalMerges((space, index) -> false); // TODO wire with stats aggregator + return new MaintenanceWhenPendingGlobalMerges(stateVersionTracker.createMergePendingChecker()); } /** diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java index 7232b2f9346..2063bdb24c9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java @@ -144,6 +144,10 @@ public class StateVersionTracker { return clusterStatsChangeTracker.statsHaveChanged(); } + MergePendingChecker createMergePendingChecker() { + return clusterStateView.getStatsAggregator().createMergePendingChecker(); + } + /* TODO test and implement - derived default space down-condition can only _keep_ a node in maintenance (down), not transition it from up -> maintenance diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 0b54b464da0..35975c5fcdc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -602,6 +602,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri builder.min_distributor_up_ratio(0); builder.min_storage_up_ratio(0); } + builder.enable_multiple_bucket_spaces(enableMultipleBucketSpaces); } @Override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 67b084e4ce6..870d37bf4e2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -138,6 +139,15 @@ public class ContentSearchClusterTest { return new BucketspacesConfig(builder); } + private static FleetcontrollerConfig getFleetcontrollerConfig(ContentCluster cluster) { + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + cluster.getConfig(builder); + builder.cluster_name("unknown"); + builder.index(0); + builder.zookeeper_server("unknown"); + return new FleetcontrollerConfig(builder); + } + private static void assertDocumentType(String expName, String expBucketSpace, BucketspacesConfig.Documenttype docType) { assertEquals(expName, docType.name()); assertEquals(expBucketSpace, docType.bucketspace()); @@ -155,11 +165,17 @@ public class ContentSearchClusterTest { @Test public void require_that_multiple_bucket_spaces_can_be_enabled() throws Exception { - BucketspacesConfig config = getBucketspacesConfig(createClusterWithMultipleBucketSpacesEnabled()); - assertEquals(2, config.documenttype().size()); - assertDocumentType("global", "global", config.documenttype(0)); - assertDocumentType("regular", "default", config.documenttype(1)); - assertTrue(config.enable_multiple_bucket_spaces()); + ContentCluster cluster = createClusterWithMultipleBucketSpacesEnabled(); + { + BucketspacesConfig config = getBucketspacesConfig(cluster); + assertEquals(2, config.documenttype().size()); + assertDocumentType("global", "global", config.documenttype(0)); + assertDocumentType("regular", "default", config.documenttype(1)); + assertTrue(config.enable_multiple_bucket_spaces()); + } + { + assertTrue(getFleetcontrollerConfig(cluster).enable_multiple_bucket_spaces()); + } } } diff --git a/configdefinitions/src/vespa/fleetcontroller.def b/configdefinitions/src/vespa/fleetcontroller.def index 028bece1db2..c9d6a958eda 100644 --- a/configdefinitions/src/vespa/fleetcontroller.def +++ b/configdefinitions/src/vespa/fleetcontroller.def @@ -153,4 +153,7 @@ min_node_ratio_per_group double default=0.0 ## This config specifies the maximum time a task can be held in this queue ## before being automatically failed out, if a version has not been ACKed ## within this duration. -max_deferred_task_version_wait_time_sec double default=30.0
\ No newline at end of file +max_deferred_task_version_wait_time_sec double default=30.0 + +## Switch to enable multiple bucket spaces in cluster controller. +enable_multiple_bucket_spaces bool default=false |