diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2021-02-04 15:24:58 +0100 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2021-02-04 15:33:43 +0000 |
commit | c4aaa09747410a5063d5f68433c892750745e184 (patch) | |
tree | 053ae49610a67953e3022498fdcd873d45e01edc /clustercontroller-core | |
parent | a07e2ca9a1ee2b56319f038b33bf03d2a5cd4e2f (diff) |
Add resource usage per node to cluster controller status page
Also adds top-level cluster feed block status. Does not yet make
enum store/multivalue limit feed blocks visible per node.
Diffstat (limited to 'clustercontroller-core')
3 files changed, 56 insertions, 1 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java index 06d728d5625..01ec8d65c28 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java @@ -14,6 +14,7 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; +import com.yahoo.vespa.clustercontroller.core.status.statuspage.HtmlTable; import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRenderer; import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; @@ -53,6 +54,7 @@ public class ContentCluster { setNodes(configuredNodes); } + // TODO move out, this doesn't belong in a domain model class public void writeHtmlState( final VdsClusterHtmlRenderer vdsClusterHtmlRenderer, final StringBuilder sb, @@ -66,6 +68,12 @@ public class ContentCluster { final VdsClusterHtmlRenderer.Table table = vdsClusterHtmlRenderer.createNewClusterHtmlTable(clusterName, slobrokGenerationCount); + if (state.clusterFeedIsBlocked()) { // Implies FeedBlock != null + table.appendRaw("<h3 style=\"color: red\">Cluster feeding is blocked!</h3>\n"); + table.appendRaw(String.format("<p>Summary: <strong>%s</strong></p>\n", + HtmlTable.escape(state.getFeedBlockOrNull().getDescription()))); + } + final List<Group> groups = LeafGroups.enumerateFrom(distribution.getRootGroup()); for (int j=0; j<groups.size(); ++j) { @@ -87,6 +95,7 @@ public class ContentCluster { statsAggregator, options.minMergeCompletionRatio, options.maxPrematureCrashes, + options.clusterFeedBlockLimit, eventLog, clusterName, localName); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java index 6d21aa430a2..ac4cb25a9c1 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java @@ -50,6 +50,10 @@ public class VdsClusterHtmlRenderer { .append(slobrokGenerationCount).append(".</p>\n"); } + public void appendRaw(String rawHtml) { + contentBuilder.append(rawHtml); + } + public void addTable(final StringBuilder destination, final long stableStateTimePeriode) { destination.append(contentBuilder); @@ -67,6 +71,7 @@ public class VdsClusterHtmlRenderer { final ClusterStatsAggregator statsAggregator, final double minMergeCompletionRatio, final int maxPrematureCrashes, + final Map<String, Double> feedBlockLimits, final EventLog eventLog, final String pathPrefix, final String name) { @@ -80,6 +85,7 @@ public class VdsClusterHtmlRenderer { statsAggregator, minMergeCompletionRatio, maxPrematureCrashes, + feedBlockLimits, eventLog, pathPrefix, dominantVtag, @@ -91,6 +97,7 @@ public class VdsClusterHtmlRenderer { statsAggregator, minMergeCompletionRatio, maxPrematureCrashes, + feedBlockLimits, eventLog, pathPrefix, dominantVtag, @@ -144,6 +151,8 @@ public class VdsClusterHtmlRenderer { .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1))) .addCell(new HtmlTable.Cell(FixedBucketSpaces.globalSpace() + " buckets") .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1))) + .addCell(new HtmlTable.Cell("Resource usage (%)") + .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1))) .addCell(new HtmlTable.Cell("Start Time")) .addCell(new HtmlTable.Cell("RPC Address"))); table.addRow(new HtmlTable.Row().setHeaderRow().addProperties(headerProperties) @@ -153,7 +162,9 @@ public class VdsClusterHtmlRenderer { .addCell(new HtmlTable.Cell("Pending")) .addCell(new HtmlTable.Cell("Total")) .addCell(new HtmlTable.Cell("Pending")) - .addCell(new HtmlTable.Cell("Total"))); + .addCell(new HtmlTable.Cell("Total")) + .addCell(new HtmlTable.Cell("Disk")) + .addCell(new HtmlTable.Cell("Memory"))); } private void renderNodesOneType( @@ -164,6 +175,7 @@ public class VdsClusterHtmlRenderer { final ClusterStatsAggregator statsAggregator, final double minMergeCompletionRatio, final int maxPrematureCrashes, + final Map<String, Double> feedBlockLimits, final EventLog eventLog, final String pathPrefix, final String dominantVtag, @@ -188,6 +200,7 @@ public class VdsClusterHtmlRenderer { addPrematureCrashes(maxPrematureCrashes, nodeInfo, row); addEventsLastWeek(eventLog, currentTime, nodeInfo, row); addBucketSpacesStats(nodeType, statsAggregator, minMergeCompletionRatio, nodeInfo, row); + addResourceUsage(nodeInfo, feedBlockLimits, row); addStartTime(nodeInfo, row); addRpcAddress(nodeInfo, row); @@ -230,6 +243,37 @@ public class VdsClusterHtmlRenderer { } } + private void addResourceUsage(NodeInfo nodeInfo, Map<String, Double> feedBlockLimits, HtmlTable.Row row) { + if (nodeInfo.isDistributor()) { + row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY)); + row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY)); + return; + } + addSingleResourceUsageCell(nodeInfo, "disk", feedBlockLimits, row); + addSingleResourceUsageCell(nodeInfo, "memory", feedBlockLimits, row); + } + + private void addSingleResourceUsageCell(NodeInfo nodeInfo, String resourceType, + Map<String, Double> feedBlockLimits, HtmlTable.Row row) + { + var hostInfo = nodeInfo.getHostInfo(); + var usages = hostInfo.getContentNode().getResourceUsage(); + + var usage = usages.get(resourceType); + if (usage != null && usage.getUsage() != null) { + row.addCell(new HtmlTable.Cell(String.format("%.2f", usage.getUsage() * 100.0))); + double limit = feedBlockLimits.getOrDefault(resourceType, 1.0); + // Mark as error if limit exceeded, warn if within 5% of exceeding + if (usage.getUsage() > limit) { + row.getLastCell().addProperties(ERROR_PROPERTY); + } else if (usage.getUsage() > (limit - 0.05)) { + row.getLastCell().addProperties(WARNING_PROPERTY); + } + } else { + row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY)); + } + } + private void addEventsLastWeek(EventLog eventLog, long currentTime, NodeInfo nodeInfo, HtmlTable.Row row) { int nodeEvents = eventLog.getNodeEventsSince(nodeInfo.getNode(), currentTime - eventLog.getRecentTimePeriod()); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java index 11b07c037ae..3042e323879 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java @@ -12,6 +12,7 @@ import org.junit.Test; import java.io.StringWriter; import java.io.Writer; +import java.util.Collections; import java.util.TreeMap; import static org.hamcrest.core.Is.is; @@ -60,6 +61,7 @@ public class ContentClusterHtmlRendrerTest { statsAggregator, 1.0, 10, + Collections.emptyMap(), eventLog, "pathPrefix", "name"); |