diff options
author | Geir Storli <geirst@oath.com> | 2018-02-28 17:20:13 +0100 |
---|---|---|
committer | Geir Storli <geirst@oath.com> | 2018-02-28 17:20:13 +0100 |
commit | a4a1f1d7cf3b7f2cbff6ec4c623fe6eac1ee724a (patch) | |
tree | cd5bf5a5946c8155c5b7a68c46794592651001f5 /clustercontroller-core | |
parent | 5428945ee928e6ac2536ebaadb8d35475d00bb10 (diff) |
Add per node buckets pending stats on html status page.
Diffstat (limited to 'clustercontroller-core')
6 files changed, 93 insertions, 34 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java index 74ddd941afb..27612bb8b7d 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java @@ -46,6 +46,7 @@ public class ContentCluster { final StringBuilder sb, final Timer timer, final ClusterState state, + final ClusterStatsAggregator statsAggregator, final Distribution distribution, final FleetControllerOptions options, final EventLog eventLog) { @@ -71,6 +72,7 @@ public class ContentCluster { distributorNodeInfoByIndex, timer, state, + statsAggregator, options.maxPrematureCrashes, eventLog, clusterName, diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java index 2698b079073..bc476923b23 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java @@ -29,11 +29,11 @@ public class ContentClusterStats implements Iterable<ContentNodeStats> { return mapToNodeStats.values().iterator(); } - ContentNodeStats getContentNode(Integer index) { + public ContentNodeStats getContentNode(Integer index) { return mapToNodeStats.get(index); } - int size() { + public int size() { return mapToNodeStats.size(); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java index 2063bdb24c9..1a57975c4bc 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java @@ -78,6 +78,10 @@ public class StateVersionTracker { return currentClusterState.getBaselineClusterState(); } + public ClusterStatsAggregator getAggregatedClusterStats() { + return clusterStateView.getStatsAggregator(); + } + public ClusterStateBundle getVersionedClusterStateBundle() { return currentClusterState; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java index 2ced8648ced..c7f83743708 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java @@ -65,6 +65,7 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa content, timer, stateVersionTracker.getVersionedClusterState(), + stateVersionTracker.getAggregatedClusterStats(), data.getOptions().storageDistribution, data.getOptions(), eventLog diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java index 2ed98855121..2544ac9e07c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.status.statuspage; +import com.yahoo.document.FixedBucketSpaces; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; -import com.yahoo.vespa.clustercontroller.core.EventLog; -import com.yahoo.vespa.clustercontroller.core.NodeInfo; -import com.yahoo.vespa.clustercontroller.core.RealTimer; -import com.yahoo.vespa.clustercontroller.core.Timer; +import com.yahoo.vespa.clustercontroller.core.*; import java.util.ArrayList; import java.util.HashMap; @@ -30,6 +28,9 @@ public class VdsClusterHtmlRendrer { private final HtmlTable.CellProperties headerProperties; private final StringBuilder contentBuilder = new StringBuilder(); private final static String TAG_NOT_SET = "not set"; + private final static HtmlTable.CellProperties WARNING_PROPERTY = new HtmlTable.CellProperties().setBackgroundColor(0xffffc0); + private final static HtmlTable.CellProperties ERROR_PROPERTY = new HtmlTable.CellProperties().setBackgroundColor(0xffc0c0); + private final static HtmlTable.CellProperties CENTERED_PROPERTY = new HtmlTable.CellProperties().align(HtmlTable.Orientation.CENTER); Table(final String clusterName, final int slobrokGenerationCount) { table.getTableProperties().align(HtmlTable.Orientation.RIGHT).setBackgroundColor(0xc0ffc0); @@ -38,8 +39,8 @@ public class VdsClusterHtmlRendrer { table.getColProperties(2).align(HtmlTable.Orientation.LEFT); table.getColProperties(3).align(HtmlTable.Orientation.LEFT); table.getColProperties(7).align(HtmlTable.Orientation.LEFT); - table.getColProperties(12).align(HtmlTable.Orientation.LEFT); - for (int i = 4; i < 13; ++i) table.getColProperties(i).allowLineBreaks(false); + table.getColProperties(14).align(HtmlTable.Orientation.LEFT); + for (int i = 4; i < 15; ++i) table.getColProperties(i).allowLineBreaks(false); headerProperties = new HtmlTable.CellProperties() .setBackgroundColor(0xffffff) .align(HtmlTable.Orientation.CENTER); @@ -64,6 +65,7 @@ public class VdsClusterHtmlRendrer { final TreeMap<Integer, NodeInfo> distributorNodeInfos, final Timer timer, final ClusterState state, + final ClusterStatsAggregator statsAggregator, final int maxPrematureCrashes, final EventLog eventLog, final String pathPrefix, @@ -75,6 +77,7 @@ public class VdsClusterHtmlRendrer { NodeType.STORAGE, timer, state, + statsAggregator, maxPrematureCrashes, eventLog, pathPrefix, @@ -84,6 +87,7 @@ public class VdsClusterHtmlRendrer { NodeType.DISTRIBUTOR, timer, state, + statsAggregator, maxPrematureCrashes, eventLog, pathPrefix, @@ -134,12 +138,16 @@ public class VdsClusterHtmlRendrer { .addCell(new HtmlTable.Cell("SSV<sup>4)</sup>")) .addCell(new HtmlTable.Cell("PC<sup>5)</sup>")) .addCell(new HtmlTable.Cell("ELW<sup>6)</sup>")) + .addCell(new HtmlTable.Cell("Buckets pending") + .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1))) .addCell(new HtmlTable.Cell("Start Time")) .addCell(new HtmlTable.Cell("RPC Address"))); table.addRow(new HtmlTable.Row().setHeaderRow().addProperties(headerProperties) .addCell(new HtmlTable.Cell("Reported")) .addCell(new HtmlTable.Cell("Wanted")) - .addCell(new HtmlTable.Cell("System"))); + .addCell(new HtmlTable.Cell("System")) + .addCell(new HtmlTable.Cell(FixedBucketSpaces.defaultSpace())) + .addCell(new HtmlTable.Cell(FixedBucketSpaces.globalSpace()))); } private void renderNodesOneType( @@ -147,6 +155,7 @@ public class VdsClusterHtmlRendrer { final NodeType nodeType, final Timer timer, final ClusterState state, + final ClusterStatsAggregator statsAggregator, final int maxPrematureCrashes, final EventLog eventLog, final String pathPrefix, @@ -156,9 +165,6 @@ public class VdsClusterHtmlRendrer { addTableHeader(name, nodeType); for (final NodeInfo nodeInfo : nodeInfos.values()) { HtmlTable.Row row = new HtmlTable.Row(); - HtmlTable.CellProperties warning = new HtmlTable.CellProperties().setBackgroundColor(0xffffc0); - HtmlTable.CellProperties error = new HtmlTable.CellProperties().setBackgroundColor(0xffc0c0); - HtmlTable.CellProperties centered = new HtmlTable.CellProperties().align(HtmlTable.Orientation.CENTER); // Add node index row.addCell(new HtmlTable.Cell("<a href=\"" + pathPrefix + "/node=" + nodeInfo.getNode() @@ -168,18 +174,18 @@ public class VdsClusterHtmlRendrer { NodeState reportedState = nodeInfo.getReportedState().clone().setStartTimestamp(0); row.addCell(new HtmlTable.Cell(HtmlTable.escape(reportedState.toString(true)))); if (!nodeInfo.getReportedState().getState().equals(State.UP)) { - row.getLastCell().addProperties(warning); + row.getLastCell().addProperties(WARNING_PROPERTY); } // Add wanted state if (nodeInfo.getWantedState() == null || nodeInfo.getWantedState().getState().equals(State.UP)) { - row.addCell(new HtmlTable.Cell("-").addProperties(centered)); + row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY)); } else { row.addCell(new HtmlTable.Cell(HtmlTable.escape(nodeInfo.getWantedState().toString(true)))); if (nodeInfo.getWantedState().toString(true).indexOf("Disabled by fleet controller") != -1) { - row.getLastCell().addProperties(error); + row.getLastCell().addProperties(ERROR_PROPERTY); } else { - row.getLastCell().addProperties(warning); + row.getLastCell().addProperties(WARNING_PROPERTY); } } @@ -188,13 +194,13 @@ public class VdsClusterHtmlRendrer { if (state.getClusterState().oneOf("uir")) { row.addCell(new HtmlTable.Cell(HtmlTable.escape(ns.toString(true)))); if (ns.getState().equals(State.DOWN)) { - row.getLastCell().addProperties(error); + row.getLastCell().addProperties(ERROR_PROPERTY); } else if (ns.getState().oneOf("mi")) { - row.getLastCell().addProperties(warning); + row.getLastCell().addProperties(WARNING_PROPERTY); } } else { row.addCell(new HtmlTable.Cell("Cluster " + - state.getClusterState().name().toLowerCase()).addProperties(error)); + state.getClusterState().name().toLowerCase()).addProperties(ERROR_PROPERTY)); } // Add build tag version. @@ -204,7 +210,7 @@ public class VdsClusterHtmlRendrer { : TAG_NOT_SET; row.addCell(new HtmlTable.Cell(buildTagText)); if (! dominantVtag.equals(nodeInfo.getVtag())) { - row.getLastCell().addProperties(warning); + row.getLastCell().addProperties(WARNING_PROPERTY); } // Add failed connection attempt count @@ -212,22 +218,22 @@ public class VdsClusterHtmlRendrer { long timeSinceContact = nodeInfo.getTimeOfFirstFailingConnectionAttempt() == 0 ? 0 : currentTime - nodeInfo.getTimeOfFirstFailingConnectionAttempt(); if (timeSinceContact > 60 * 1000) { - row.getLastCell().addProperties(error); + row.getLastCell().addProperties(ERROR_PROPERTY); } else if (nodeInfo.getConnectionAttemptCount() > 0) { - row.getLastCell().addProperties(warning); + row.getLastCell().addProperties(WARNING_PROPERTY); } // Add time since first failing row.addCell(new HtmlTable.Cell((timeSinceContact / 1000) + " s")); if (timeSinceContact > 60 * 1000) { - row.getLastCell().addProperties(error); + row.getLastCell().addProperties(ERROR_PROPERTY); } else if (nodeInfo.getConnectionAttemptCount() > 0) { - row.getLastCell().addProperties(warning); + row.getLastCell().addProperties(WARNING_PROPERTY); } // State pending time if (nodeInfo.getLatestNodeStateRequestTime() == null) { - row.addCell(new HtmlTable.Cell("-").addProperties(centered)); + row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY)); } else { row.addCell(new HtmlTable.Cell(HtmlTable.escape(RealTimer.printDuration( currentTime - nodeInfo.getLatestNodeStateRequestTime())))); @@ -236,17 +242,17 @@ public class VdsClusterHtmlRendrer { // System state version row.addCell(new HtmlTable.Cell("" + nodeInfo.getSystemStateVersionAcknowledged())); if (nodeInfo.getSystemStateVersionAcknowledged() < state.getVersion() - 2) { - row.getLastCell().addProperties(error); + row.getLastCell().addProperties(ERROR_PROPERTY); } else if (nodeInfo.getSystemStateVersionAcknowledged() < state.getVersion()) { - row.getLastCell().addProperties(warning); + row.getLastCell().addProperties(WARNING_PROPERTY); } // Premature crashes row.addCell(new HtmlTable.Cell("" + nodeInfo.getPrematureCrashCount())); if (nodeInfo.getPrematureCrashCount() >= maxPrematureCrashes) { - row.getLastCell().addProperties(error); + row.getLastCell().addProperties(ERROR_PROPERTY); } else if (nodeInfo.getPrematureCrashCount() > 0) { - row.getLastCell().addProperties(warning); + row.getLastCell().addProperties(WARNING_PROPERTY); } // Events last week @@ -254,14 +260,23 @@ public class VdsClusterHtmlRendrer { currentTime - eventLog.getRecentTimePeriod()); row.addCell(new HtmlTable.Cell("" + nodeEvents)); if (nodeEvents > 20) { - row.getLastCell().addProperties(error); + row.getLastCell().addProperties(ERROR_PROPERTY); } else if (nodeEvents > 3) { - row.getLastCell().addProperties(warning); + row.getLastCell().addProperties(WARNING_PROPERTY); + } + + // Buckets pending ('default' and 'global' spaces) + if (nodeType.equals(NodeType.STORAGE)) { + addBucketsPending(row, getStatsForContentNode(statsAggregator, nodeInfo, FixedBucketSpaces.defaultSpace())); + addBucketsPending(row, getStatsForContentNode(statsAggregator, nodeInfo, FixedBucketSpaces.globalSpace())); + } else { + addBucketsPending(row, getStatsForDistributorNode(statsAggregator, nodeInfo, FixedBucketSpaces.defaultSpace())); + addBucketsPending(row, getStatsForDistributorNode(statsAggregator, nodeInfo, FixedBucketSpaces.globalSpace())); } // Start time if (nodeInfo.getStartTimestamp() == 0) { - row.addCell(new HtmlTable.Cell("-").addProperties(error).addProperties(centered)); + row.addCell(new HtmlTable.Cell("-").addProperties(ERROR_PROPERTY).addProperties(CENTERED_PROPERTY)); } else { String startTime = RealTimer.printDateNoMilliSeconds( 1000 * nodeInfo.getStartTimestamp(), utcTimeZone); @@ -270,16 +285,50 @@ public class VdsClusterHtmlRendrer { // RPC address if (nodeInfo.getRpcAddress() == null) { - row.addCell(new HtmlTable.Cell("-").addProperties(error)); + row.addCell(new HtmlTable.Cell("-").addProperties(ERROR_PROPERTY)); } else { row.addCell(new HtmlTable.Cell(HtmlTable.escape(nodeInfo.getRpcAddress()))); if (nodeInfo.isRpcAddressOutdated()) { - row.getLastCell().addProperties(warning); + row.getLastCell().addProperties(WARNING_PROPERTY); } } table.addRow(row); } } + + private static ContentNodeStats.BucketSpaceStats getStatsForContentNode(ClusterStatsAggregator statsAggregator, + NodeInfo nodeInfo, + String bucketSpace) { + ContentNodeStats nodeStats = statsAggregator.getAggregatedStats().getContentNode(nodeInfo.getNodeIndex()); + if (nodeStats != null) { + return nodeStats.getBucketSpace(bucketSpace); + } + return null; + } + + private static ContentNodeStats.BucketSpaceStats getStatsForDistributorNode(ClusterStatsAggregator statsAggregator, + NodeInfo nodeInfo, + String bucketSpace) { + ContentNodeStats nodeStats = statsAggregator.getAggregatedStatsForDistributor(nodeInfo.getNodeIndex()); + return nodeStats.getBucketSpace(bucketSpace); + } + + private static void addBucketsPending(HtmlTable.Row row, ContentNodeStats.BucketSpaceStats bucketSpaceStats) { + if (bucketSpaceStats != null) { + long bucketsPending = bucketSpaceStats.getBucketsPending(); + String cellValue = "" + bucketsPending; + if (!bucketSpaceStats.valid()) { + cellValue += "?"; + } + row.addCell(new HtmlTable.Cell(cellValue)); + if (bucketsPending > 0 || !bucketSpaceStats.valid()) { + row.getLastCell().addProperties(WARNING_PROPERTY); + } + } else { + row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY)); + } + } + private void addFooter(final StringBuilder contentBuilder, final long stableStateTimePeriode) { contentBuilder.append("<font size=\"-1\">\n") .append("1) FC - Failed connections - We have tried to connect to the nodes this many times " + diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java index 24d387f65de..6dc260d6bf3 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.google.common.collect.Sets; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRendrer; @@ -48,12 +49,14 @@ public class ContentClusterHtmlRendrerTest { distributorNodeInfoByIndex.put(x, nodeInfo); } storageNodeInfoByIndex.put(2, new StorageNodeInfo(contentCluster, 2, false, "storage" + 2, null)); + ClusterStatsAggregator statsAggregator = new ClusterStatsAggregator(Sets.newHashSet(2), Sets.newHashSet(2)); table.renderNodes( storageNodeInfoByIndex, distributorNodeInfoByIndex, new FakeTimer(), state, + statsAggregator, 10, eventLog, "pathPrefix", |