From dafd2b5b69a9704858b5da0f5066c2f0a89951a7 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Tue, 20 Mar 2018 14:48:55 +0100 Subject: Add config to clustercontroller for the min merge completion ratio of buckets in a bucket space before it is considered complete. --- .../core/AggregatedStatsMergePendingChecker.java | 12 ++++++------ .../clustercontroller/core/ClusterStatsAggregator.java | 4 ++-- .../core/ClusterStatsChangeTracker.java | 15 +++++++++------ .../vespa/clustercontroller/core/ContentNodeStats.java | 14 ++++++++++++-- .../vespa/clustercontroller/core/FleetController.java | 3 ++- .../clustercontroller/core/FleetControllerOptions.java | 3 +++ .../clustercontroller/core/StateVersionTracker.java | 16 ++++++++++++---- 7 files changed, 46 insertions(+), 21 deletions(-) (limited to 'clustercontroller-core/src/main/java/com') diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AggregatedStatsMergePendingChecker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AggregatedStatsMergePendingChecker.java index 07586569bf5..e6ce1df2b48 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AggregatedStatsMergePendingChecker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AggregatedStatsMergePendingChecker.java @@ -1,10 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.document.FixedBucketSpaces; - -import java.util.Iterator; - /** * Class checking whether a particular bucket space on a content node might have buckets pending. * @@ -13,9 +9,12 @@ import java.util.Iterator; public class AggregatedStatsMergePendingChecker implements MergePendingChecker { private final AggregatedClusterStats stats; + private final double minMergeCompletionRatio; - public AggregatedStatsMergePendingChecker(AggregatedClusterStats stats) { + public AggregatedStatsMergePendingChecker(AggregatedClusterStats stats, + double minMergeCompletionRatio) { this.stats = stats; + this.minMergeCompletionRatio = minMergeCompletionRatio; } @Override @@ -26,7 +25,8 @@ public class AggregatedStatsMergePendingChecker implements MergePendingChecker { ContentNodeStats nodeStats = stats.getStats().getContentNode(contentNodeIndex); if (nodeStats != null) { ContentNodeStats.BucketSpaceStats bucketSpaceStats = nodeStats.getBucketSpace(bucketSpace); - return (bucketSpaceStats != null && bucketSpaceStats.mayHaveBucketsPending()); + return (bucketSpaceStats != null && + bucketSpaceStats.mayHaveBucketsPending(minMergeCompletionRatio)); } return true; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java index 9db4ce4e44a..2a9ca13b3ee 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java @@ -73,8 +73,8 @@ public class ClusterStatsAggregator { return result; } - MergePendingChecker createMergePendingChecker() { - return new AggregatedStatsMergePendingChecker(getAggregatedStats()); + MergePendingChecker createMergePendingChecker(double minMergeCompletionRatio) { + return new AggregatedStatsMergePendingChecker(getAggregatedStats(), minMergeCompletionRatio); } /** diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsChangeTracker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsChangeTracker.java index 75e8db40f0e..68f19c721d9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsChangeTracker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsChangeTracker.java @@ -19,13 +19,15 @@ public class ClusterStatsChangeTracker { private AggregatedStatsMergePendingChecker checker; private Map prevMayHaveMergesPending = null; - public ClusterStatsChangeTracker(AggregatedClusterStats aggregatedStats) { - setAggregatedStats(aggregatedStats); + public ClusterStatsChangeTracker(AggregatedClusterStats aggregatedStats, + double minMergeCompletionRatio) { + setAggregatedStats(aggregatedStats, minMergeCompletionRatio); } - private void setAggregatedStats(AggregatedClusterStats aggregatedStats) { + private void setAggregatedStats(AggregatedClusterStats aggregatedStats, + double minMergeCompletionRatio) { this.aggregatedStats = aggregatedStats; - checker = new AggregatedStatsMergePendingChecker(this.aggregatedStats); + checker = new AggregatedStatsMergePendingChecker(this.aggregatedStats, minMergeCompletionRatio); } public void syncAggregatedStats() { @@ -36,9 +38,10 @@ public class ClusterStatsChangeTracker { } } - public void updateAggregatedStats(AggregatedClusterStats newAggregatedStats) { + public void updateAggregatedStats(AggregatedClusterStats newAggregatedStats, + double minMergeCompletionRatio) { syncAggregatedStats(); - setAggregatedStats(newAggregatedStats); + setAggregatedStats(newAggregatedStats, minMergeCompletionRatio); } public boolean statsHaveChanged() { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStats.java index 72a7b2e4bcf..8bbbb20bef4 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStats.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStats.java @@ -54,8 +54,18 @@ public class ContentNodeStats { return bucketsPending; } - public boolean mayHaveBucketsPending() { - return (bucketsPending > 0) || (invalidCount > 0); + public boolean mayHaveBucketsPending(double minMergeCompletionRatio) { + if (invalidCount > 0) { + return true; + } + if (bucketsTotal == 0) { + return bucketsPending > 0; + } + return calcMergeCompletionRatio() < minMergeCompletionRatio; + } + + private double calcMergeCompletionRatio() { + return ((double) Math.max(0, (bucketsTotal - bucketsPending)) / (double) bucketsTotal); } public boolean valid() { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 2e2534916a7..54c6c11f81e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -121,7 +121,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd this.stateGatherer = nodeStateGatherer; this.stateChangeHandler = stateChangeHandler; this.systemStateBroadcaster = systemStateBroadcaster; - this.stateVersionTracker = new StateVersionTracker(); + this.stateVersionTracker = new StateVersionTracker(options.minMergeCompletionRatio); this.metricUpdater = metricUpdater; this.statusPageServer = statusPage; @@ -452,6 +452,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } else { configuredBucketSpaces = Collections.emptySet(); } + stateVersionTracker.setMinMergeCompletionRatio(options.minMergeCompletionRatio); communicator.propagateOptions(options); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java index 3bf8edafbe3..860d38b3438 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java @@ -122,6 +122,9 @@ public class FleetControllerOptions implements Cloneable { // TODO replace this flag with a set of bucket spaces instead public boolean enableMultipleBucketSpaces = false; + // TODO: Choose a default value + public double minMergeCompletionRatio = 1.0; + // TODO: Replace usage of this by usage where the nodes are explicitly passed (below) public FleetControllerOptions(String clusterName) { this.clusterName = clusterName; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java index d4cd8f902c6..0673a10c282 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java @@ -36,10 +36,13 @@ public class StateVersionTracker { private final LinkedList clusterStateHistory = new LinkedList<>(); private int maxHistoryEntryCount = 50; + private double minMergeCompletionRatio; - StateVersionTracker() { + StateVersionTracker(double minMergeCompletionRatio) { clusterStateView = ClusterStateView.create(currentUnversionedState.getBaselineClusterState()); - clusterStatsChangeTracker = new ClusterStatsChangeTracker(clusterStateView.getStatsAggregator().getAggregatedStats()); + clusterStatsChangeTracker = new ClusterStatsChangeTracker(clusterStateView.getStatsAggregator().getAggregatedStats(), + minMergeCompletionRatio); + this.minMergeCompletionRatio = minMergeCompletionRatio; } void setVersionRetrievedFromZooKeeper(final int version) { @@ -58,6 +61,10 @@ public class StateVersionTracker { this.maxHistoryEntryCount = maxHistoryEntryCount; } + void setMinMergeCompletionRatio(double minMergeCompletionRatio) { + this.minMergeCompletionRatio = minMergeCompletionRatio; + } + int getCurrentVersion() { return this.currentVersion; } @@ -128,7 +135,8 @@ public class StateVersionTracker { newStateBundle.getBaselineClusterState().getDistributionBitCount()); // TODO should this take place in updateLatestCandidateStateBundle instead? I.e. does it require a consolidated state? clusterStateView = ClusterStateView.create(currentClusterState.getBaselineClusterState()); - clusterStatsChangeTracker.updateAggregatedStats(clusterStateView.getStatsAggregator().getAggregatedStats()); + clusterStatsChangeTracker.updateAggregatedStats(clusterStateView.getStatsAggregator().getAggregatedStats(), + minMergeCompletionRatio); } private void recordCurrentStateInHistoryAtTime(final long currentTimeMs) { @@ -149,7 +157,7 @@ public class StateVersionTracker { } MergePendingChecker createMergePendingChecker() { - return clusterStateView.getStatsAggregator().createMergePendingChecker(); + return clusterStateView.getStatsAggregator().createMergePendingChecker(minMergeCompletionRatio); } /* -- cgit v1.2.3 From ee445011cf66284cd20600a2060f8a0933ad3c5f Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Tue, 20 Mar 2018 15:17:01 +0100 Subject: Add buckets total to per bucket space stats on html status page. --- .../clustercontroller/core/ContentCluster.java | 1 + .../status/statuspage/VdsClusterHtmlRendrer.java | 44 +++++++++++++++------- .../core/ContentClusterHtmlRendrerTest.java | 1 + 3 files changed, 33 insertions(+), 13 deletions(-) (limited to 'clustercontroller-core/src/main/java/com') diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java index 27612bb8b7d..6ff297b4a31 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java @@ -73,6 +73,7 @@ public class ContentCluster { timer, state, statsAggregator, + options.minMergeCompletionRatio, options.maxPrematureCrashes, eventLog, clusterName, diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java index 80aed35cfa1..551eb34f8fa 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java @@ -66,6 +66,7 @@ public class VdsClusterHtmlRendrer { final Timer timer, final ClusterState state, final ClusterStatsAggregator statsAggregator, + final double minMergeCompletionRatio, final int maxPrematureCrashes, final EventLog eventLog, final String pathPrefix, @@ -78,6 +79,7 @@ public class VdsClusterHtmlRendrer { timer, state, statsAggregator, + minMergeCompletionRatio, maxPrematureCrashes, eventLog, pathPrefix, @@ -88,6 +90,7 @@ public class VdsClusterHtmlRendrer { timer, state, statsAggregator, + minMergeCompletionRatio, maxPrematureCrashes, eventLog, pathPrefix, @@ -138,7 +141,9 @@ public class VdsClusterHtmlRendrer { .addCell(new HtmlTable.Cell("SSV4)")) .addCell(new HtmlTable.Cell("PC5)")) .addCell(new HtmlTable.Cell("ELW6)")) - .addCell(new HtmlTable.Cell("Buckets pending") + .addCell(new HtmlTable.Cell(FixedBucketSpaces.defaultSpace() + " buckets") + .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1))) + .addCell(new HtmlTable.Cell(FixedBucketSpaces.globalSpace() + " buckets") .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1))) .addCell(new HtmlTable.Cell("Start Time")) .addCell(new HtmlTable.Cell("RPC Address"))); @@ -146,8 +151,10 @@ public class VdsClusterHtmlRendrer { .addCell(new HtmlTable.Cell("Reported")) .addCell(new HtmlTable.Cell("Wanted")) .addCell(new HtmlTable.Cell("System")) - .addCell(new HtmlTable.Cell(FixedBucketSpaces.defaultSpace())) - .addCell(new HtmlTable.Cell(FixedBucketSpaces.globalSpace()))); + .addCell(new HtmlTable.Cell("Pending")) + .addCell(new HtmlTable.Cell("Total")) + .addCell(new HtmlTable.Cell("Pending")) + .addCell(new HtmlTable.Cell("Total"))); } private void renderNodesOneType( @@ -156,6 +163,7 @@ public class VdsClusterHtmlRendrer { final Timer timer, final ClusterState state, final ClusterStatsAggregator statsAggregator, + final double minMergeCompletionRatio, final int maxPrematureCrashes, final EventLog eventLog, final String pathPrefix, @@ -265,13 +273,17 @@ public class VdsClusterHtmlRendrer { row.getLastCell().addProperties(WARNING_PROPERTY); } - // Buckets pending ('default' and 'global' spaces) + // Bucket stats for 'default' and 'global' spaces if (nodeType.equals(NodeType.STORAGE)) { - addBucketsPending(row, getStatsForContentNode(statsAggregator, nodeInfo, FixedBucketSpaces.defaultSpace())); - addBucketsPending(row, getStatsForContentNode(statsAggregator, nodeInfo, FixedBucketSpaces.globalSpace())); + addBucketStats(row, getStatsForContentNode(statsAggregator, nodeInfo, FixedBucketSpaces.defaultSpace()), + minMergeCompletionRatio); + addBucketStats(row, getStatsForContentNode(statsAggregator, nodeInfo, FixedBucketSpaces.globalSpace()), + minMergeCompletionRatio); } else { - addBucketsPending(row, getStatsForDistributorNode(statsAggregator, nodeInfo, FixedBucketSpaces.defaultSpace())); - addBucketsPending(row, getStatsForDistributorNode(statsAggregator, nodeInfo, FixedBucketSpaces.globalSpace())); + addBucketStats(row, getStatsForDistributorNode(statsAggregator, nodeInfo, FixedBucketSpaces.defaultSpace()), + minMergeCompletionRatio); + addBucketStats(row, getStatsForDistributorNode(statsAggregator, nodeInfo, FixedBucketSpaces.globalSpace()), + minMergeCompletionRatio); } // Start time @@ -313,19 +325,25 @@ public class VdsClusterHtmlRendrer { return nodeStats.getBucketSpace(bucketSpace); } - private static void addBucketsPending(HtmlTable.Row row, ContentNodeStats.BucketSpaceStats bucketSpaceStats) { + private static void addBucketStats(HtmlTable.Row row, ContentNodeStats.BucketSpaceStats bucketSpaceStats, + double minMergeCompletionRatio) { if (bucketSpaceStats != null) { long bucketsPending = bucketSpaceStats.getBucketsPending(); - String cellValue = String.valueOf(bucketsPending); + long bucketsTotal = bucketSpaceStats.getBucketsTotal(); + String cellValuePending = String.valueOf(bucketsPending); + String cellValueTotal = String.valueOf(bucketsTotal); if (!bucketSpaceStats.valid()) { - cellValue += "?"; + cellValuePending += "?"; + cellValueTotal += "?"; } - row.addCell(new HtmlTable.Cell(cellValue)); - if (bucketsPending > 0 || !bucketSpaceStats.valid()) { + row.addCell(new HtmlTable.Cell(cellValuePending)); + if (bucketSpaceStats.mayHaveBucketsPending(minMergeCompletionRatio)) { row.getLastCell().addProperties(WARNING_PROPERTY); } + row.addCell(new HtmlTable.Cell(cellValueTotal)); } else { row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY)); + row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY)); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java index 6dc260d6bf3..f5adf644c28 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java @@ -57,6 +57,7 @@ public class ContentClusterHtmlRendrerTest { new FakeTimer(), state, statsAggregator, + 1.0, 10, eventLog, "pathPrefix", -- cgit v1.2.3