diff options
author | Harald Musum <musum@yahooinc.com> | 2023-03-28 14:14:49 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2023-03-28 14:14:49 +0200 |
commit | a9646ed0608bf59ed831617eec559a10723fe7fa (patch) | |
tree | c37346a4a9f7def6dac613eedb156f95ad422a22 /vdslib | |
parent | c2634969650aec3c3f3744fd2069a4b5b58945ca (diff) |
Deprecate distributor_auto_ownership_transfer_on_whole_group_down
Stop using the config value (same as currrent default value, which is true).
Also: distribution config does not have this field, only stor-distribution config has
Diffstat (limited to 'vdslib')
3 files changed, 11 insertions, 649 deletions
diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java index 4f9186a0569..9eb88661002 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java @@ -26,20 +26,10 @@ import java.util.concurrent.atomic.AtomicReference; public class Distribution { - private static class Config { - Config(Group nodeGraph, int redundancy, boolean distributorAutoOwnershipTransferOnWholeGroupDown) { - this.nodeGraph = nodeGraph; - this.redundancy = redundancy; - this.distributorAutoOwnershipTransferOnWholeGroupDown = distributorAutoOwnershipTransferOnWholeGroupDown; - } - - private final Group nodeGraph; - private final int redundancy; - private final boolean distributorAutoOwnershipTransferOnWholeGroupDown; - } + private record Config(Group nodeGraph, int redundancy) { } private ConfigSubscriber configSub; - private final AtomicReference<Config> config = new AtomicReference<>(new Config(null, 1, false)); + private final AtomicReference<Config> config = new AtomicReference<>(new Config(null, 1)); public Group getRootGroup() { return config.getAcquire().nodeGraph; @@ -96,7 +86,7 @@ public class Distribution { if (root == null) throw new IllegalStateException("Config does not specify a root group"); root.calculateDistributionHashValues(); - Distribution.this.config.setRelease(new Config(root, config.redundancy(), config.distributor_auto_ownership_transfer_on_whole_group_down())); + Distribution.this.config.setRelease(new Config(root, config.redundancy())); } catch (ParseException e) { throw new IllegalStateException("Failed to parse config", e); } @@ -139,7 +129,7 @@ public class Distribution { if (root == null) throw new IllegalStateException("Config does not specify a root group"); root.calculateDistributionHashValues(); - Distribution.this.config.setRelease(new Config(root, config.redundancy(), true)); + Distribution.this.config.setRelease(new Config(root, config.redundancy())); } catch (ParseException e) { throw new IllegalStateException("Failed to parse config", e); } @@ -211,20 +201,10 @@ public class Distribution { } } - private static class ScoredNode { - final double score; - final int index; - - ScoredNode(int index, double score) { - this.score = score; - this.index = index; - } + private record ScoredNode(int index, double score) { boolean valid() { return index != -1; } - - static ScoredNode makeInvalid() { - return new ScoredNode(-1, 0.0); - } + static ScoredNode makeInvalid() { return new ScoredNode(-1, 0.0); } } private static boolean allDistributorsDown(Group g, ClusterState clusterState) { @@ -241,8 +221,7 @@ public class Distribution { return true; } - private Group getIdealDistributorGroup(boolean distributorAutoOwnershipTransferOnWholeGroupDown, - BucketId bucket, ClusterState clusterState, Group parent, int redundancy) { + private Group getIdealDistributorGroup(BucketId bucket, ClusterState clusterState, Group parent, int redundancy) { if (parent.isLeafGroup()) { return parent; } @@ -259,15 +238,13 @@ public class Distribution { } results.add(new ScoredGroup(g, score)); } - if (distributorAutoOwnershipTransferOnWholeGroupDown) { - while (!results.isEmpty() && allDistributorsDown(results.first().group, clusterState)) { - results.remove(results.first()); - } + while (!results.isEmpty() && allDistributorsDown(results.first().group, clusterState)) { + results.remove(results.first()); } if (results.isEmpty()) { return null; } - return getIdealDistributorGroup(distributorAutoOwnershipTransferOnWholeGroupDown, bucket, clusterState, results.first().group, redundancyArray[0]); + return getIdealDistributorGroup(bucket, clusterState, results.first().group, redundancyArray[0]); } private static class ResultGroup implements Comparable<ResultGroup> { @@ -434,7 +411,7 @@ public class Distribution { } Config cfg = config.getAcquire(); - Group idealGroup = getIdealDistributorGroup(cfg.distributorAutoOwnershipTransferOnWholeGroupDown, bucket, state, cfg.nodeGraph, cfg.redundancy); + Group idealGroup = getIdealDistributorGroup(bucket, state, cfg.nodeGraph, cfg.redundancy); if (idealGroup == null) { throw new NoDistributorsAvailableException("No distributors available in cluster state version " + state.getVersion()); } diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java index 415c4ffe8f0..19c9c79522d 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java @@ -345,36 +345,6 @@ public class DistributionTestCase { } @Test - public void testDistributorNoGroupTakeover() throws Exception { - test = new DistributionTestFactory("hierarchical-grouping-distributor-notakeover") - .setDistribution(buildHierarchicalConfig(6, 3, 1, "1|2|*", 3).distributor_auto_ownership_transfer_on_whole_group_down(false)) - .setNodeType(NodeType.DISTRIBUTOR) - .setClusterState(new ClusterState("distributor:2 storage:9")); - int [] counts = new int[10]; - int noneExisting = 0; - for (BucketId bucket : getTestBuckets()) { - DistributionTestFactory.Test t = test.recordResult(bucket); - List<Integer> nodes = t.getNodes(); - if (nodes.isEmpty()) { - ++noneExisting; - t.assertFailure(DistributionTestFactory.Failure.NO_DISTRIBUTORS_AVAILABLE); - } else { - t.assertNodeCount(1); - for (int i : nodes) { - ++counts[i]; - } - } - } - for (int i=2; i<10; ++i) { - assertEquals(0, counts[i]); - } - for (int i=0; i<2; ++i) { - assertTrue(counts[i] > 0); - } - assertEquals(15, noneExisting); - } - - @Test public void testHierarchicalDistributionDeep() throws Exception { System.out.println(new StorDistributionConfig(buildHierarchicalConfig(8, 5, 3, "*|*", 3))); test = new DistributionTestFactory("hierarchical-grouping-deep") diff --git a/vdslib/src/tests/distribution/testdata/hierarchical-grouping-distributor-notakeover.java.results b/vdslib/src/tests/distribution/testdata/hierarchical-grouping-distributor-notakeover.java.results deleted file mode 100644 index f092339a37a..00000000000 --- a/vdslib/src/tests/distribution/testdata/hierarchical-grouping-distributor-notakeover.java.results +++ /dev/null @@ -1,585 +0,0 @@ -{ - "cluster-state" : "distributor:2 storage:9", - "distribution" : "redundancy 6\ninitial_redundancy 0\nensure_primary_persisted true\nready_copies 0\nactive_per_leaf_group false\ndistributor_auto_ownership_transfer_on_whole_group_down false\ngroup[0].index \"invalid\"\ngroup[0].name \"invalid\"\ngroup[0].capacity 1.0\ngroup[0].partitions \"1|2|*\"\ngroup[1].index \"0\"\ngroup[1].name \"top.0\"\ngroup[1].capacity 1.0\ngroup[1].partitions \"\"\ngroup[1].nodes[0].index 8\ngroup[1].nodes[0].retired false\ngroup[1].nodes[1].index 1\ngroup[1].nodes[1].retired false\ngroup[1].nodes[2].index 6\ngroup[1].nodes[2].retired false\ngroup[2].index \"1\"\ngroup[2].name \"top.1\"\ngroup[2].capacity 1.0\ngroup[2].partitions \"\"\ngroup[2].nodes[0].index 5\ngroup[2].nodes[0].retired false\ngroup[2].nodes[1].index 0\ngroup[2].nodes[1].retired false\ngroup[2].nodes[2].index 2\ngroup[2].nodes[2].retired false\ngroup[3].index \"2\"\ngroup[3].name \"top.2\"\ngroup[3].capacity 1.0\ngroup[3].partitions \"\"\ngroup[3].nodes[0].index 4\ngroup[3].nodes[0].retired false\ngroup[3].nodes[1].index 3\ngroup[3].nodes[1].retired false\ngroup[3].nodes[2].index 7\ngroup[3].nodes[2].retired false\ndisk_distribution MODULO_BID", - "node-type" : "distributor", - "redundancy" : 3, - "node-count" : 10, - "up-states" : "uim", - "result" : [ { - "nodes" : [ 0 ], - "bucket" : "4000000000000000", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4000000000000001", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4000000000000002", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "4000000000000003", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 0 ], - "bucket" : "4000000000000004", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "4000000000000005", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4000000000000006", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4000000000000007", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4000000000000008", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "4000000000000009", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "400000000000000a", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "400000000000000b", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "400000000000000c", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ ], - "bucket" : "400000000000000d", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 1 ], - "bucket" : "400000000000000e", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "400000000000000f", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 0 ], - "bucket" : "4000000000000010", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "4000000000000011", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 0 ], - "bucket" : "4000000000000012", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4000000000000013", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "4400000000000000", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4400000000000001", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4400000000000002", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "4400000000000003", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 0 ], - "bucket" : "4400000000000004", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "4400000000000005", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4400000000000006", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4400000000000007", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4400000000000008", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "4400000000000009", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "440000000000000a", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "440000000000000b", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "440000000000000c", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ ], - "bucket" : "440000000000000d", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 1 ], - "bucket" : "440000000000000e", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "440000000000000f", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 0 ], - "bucket" : "4400000000000010", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "4400000000000011", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 0 ], - "bucket" : "4400000000000012", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4400000000000013", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "4800000000000000", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4800000000000001", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4800000000000002", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "4800000000000003", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 0 ], - "bucket" : "4800000000000004", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "4800000000000005", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4800000000000006", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4800000000000007", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4800000000000008", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "4800000000000009", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "480000000000000a", - "failure" : "NONE" - }, { - "nodes" : [ 0 ], - "bucket" : "480000000000000b", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "480000000000000c", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ ], - "bucket" : "480000000000000d", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 1 ], - "bucket" : "480000000000000e", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "480000000000000f", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 0 ], - "bucket" : "4800000000000010", - "failure" : "NONE" - }, { - "nodes" : [ ], - "bucket" : "4800000000000011", - "failure" : "NO_DISTRIBUTORS_AVAILABLE" - }, { - "nodes" : [ 0 ], - "bucket" : "4800000000000012", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4800000000000013", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "400000000000fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "440000000001fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "480000000001fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4c0000000001fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "500000000009fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "540000000009fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "580000000009fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "5c0000000049fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "600000000049fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "640000000149fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "680000000349fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "6c0000000749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "700000000749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "740000001749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "780000003749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "7c0000003749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "80000000b749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "84000001b749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "88000003b749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "8c000007b749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "9000000fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "9400001fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "9800003fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "9c00007fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "a000007fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "a400007fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "a800007fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "ac00047fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "b000047fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "b400147fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "b800347fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "bc00347fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "c000b47fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "c400b47fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "c800b47fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "cc00b47fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "d000b47fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "d400b47fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "d800b47fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "dc00b47fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "e000b47fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "e400b47fb749fe68", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "40000000000098d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "44000000000098d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "48000000000298d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "4c000000000698d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "50000000000e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "54000000000e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "58000000002e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "5c000000002e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "60000000002e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "64000000012e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "68000000012e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "6c000000052e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "70000000052e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "74000000152e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "78000000152e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "7c000000152e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "80000000952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "84000001952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "88000001952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "8c000001952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "90000001952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "94000001952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "98000001952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "9c000041952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "a00000c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "a40000c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "a80002c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "ac0002c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "b00002c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "b40012c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "b80012c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "bc0012c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "c00092c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "c40192c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "c80192c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "cc0192c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "d00992c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "d41992c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "d83992c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "dc7992c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "e07992c1952e98d2", - "failure" : "NONE" - }, { - "nodes" : [ 1 ], - "bucket" : "e47992c1952e98d2", - "failure" : "NONE" - } ] -}
\ No newline at end of file |