Add resource usage per node to cluster controller status page

Also adds top-level cluster feed block status. Does not yet make
enum store/multivalue limit feed blocks visible per node.

3 files changed, 56 insertions, 1 deletions

diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
index 06d728d5625..01ec8d65c28 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
@@ -14,6 +14,7 @@ import com.yahoo.vdslib.state.Node;
 import com.yahoo.vdslib.state.NodeState;
 import com.yahoo.vdslib.state.NodeType;
 import com.yahoo.vdslib.state.State;
+import com.yahoo.vespa.clustercontroller.core.status.statuspage.HtmlTable;
 import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRenderer;
 import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest;
 
@@ -53,6 +54,7 @@ public class ContentCluster {
         setNodes(configuredNodes);
     }
 
+    // TODO move out, this doesn't belong in a domain model class
     public void writeHtmlState(
             final VdsClusterHtmlRenderer vdsClusterHtmlRenderer,
             final StringBuilder sb,
@@ -66,6 +68,12 @@ public class ContentCluster {
         final VdsClusterHtmlRenderer.Table table =
                 vdsClusterHtmlRenderer.createNewClusterHtmlTable(clusterName, slobrokGenerationCount);
 
+        if (state.clusterFeedIsBlocked()) { // Implies FeedBlock != null
+            table.appendRaw("<h3 style=\"color: red\">Cluster feeding is blocked!</h3>\n");
+            table.appendRaw(String.format("<p>Summary: <strong>%s</strong></p>\n",
+                                          HtmlTable.escape(state.getFeedBlockOrNull().getDescription())));
+        }
+
         final List<Group> groups = LeafGroups.enumerateFrom(distribution.getRootGroup());
 
         for (int j=0; j<groups.size(); ++j) {
@@ -87,6 +95,7 @@ public class ContentCluster {
                     statsAggregator,
                     options.minMergeCompletionRatio,
                     options.maxPrematureCrashes,
+                    options.clusterFeedBlockLimit,
                     eventLog,
                     clusterName,
                     localName);
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java
index 6d21aa430a2..ac4cb25a9c1 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java
@@ -50,6 +50,10 @@ public class VdsClusterHtmlRenderer {
                     .append(slobrokGenerationCount).append(".</p>\n");
         }
 
+        public void appendRaw(String rawHtml) {
+            contentBuilder.append(rawHtml);
+        }
+
         public void addTable(final StringBuilder destination, final long stableStateTimePeriode) {
             destination.append(contentBuilder);
 
@@ -67,6 +71,7 @@ public class VdsClusterHtmlRenderer {
                 final ClusterStatsAggregator statsAggregator,
                 final double minMergeCompletionRatio,
                 final int maxPrematureCrashes,
+                final Map<String, Double> feedBlockLimits,
                 final EventLog eventLog,
                 final String pathPrefix,
                 final String name) {
@@ -80,6 +85,7 @@ public class VdsClusterHtmlRenderer {
                     statsAggregator,
                     minMergeCompletionRatio,
                     maxPrematureCrashes,
+                    feedBlockLimits,
                     eventLog,
                     pathPrefix,
                     dominantVtag,
@@ -91,6 +97,7 @@ public class VdsClusterHtmlRenderer {
                     statsAggregator,
                     minMergeCompletionRatio,
                     maxPrematureCrashes,
+                    feedBlockLimits,
                     eventLog,
                     pathPrefix,
                     dominantVtag,
@@ -144,6 +151,8 @@ public class VdsClusterHtmlRenderer {
                             .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1)))
                     .addCell(new HtmlTable.Cell(FixedBucketSpaces.globalSpace() + " buckets")
                             .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1)))
+                    .addCell(new HtmlTable.Cell("Resource usage (%)")
+                            .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1)))
                     .addCell(new HtmlTable.Cell("Start Time"))
                     .addCell(new HtmlTable.Cell("RPC Address")));
             table.addRow(new HtmlTable.Row().setHeaderRow().addProperties(headerProperties)
@@ -153,7 +162,9 @@ public class VdsClusterHtmlRenderer {
                     .addCell(new HtmlTable.Cell("Pending"))
                     .addCell(new HtmlTable.Cell("Total"))
                     .addCell(new HtmlTable.Cell("Pending"))
-                    .addCell(new HtmlTable.Cell("Total")));
+                    .addCell(new HtmlTable.Cell("Total"))
+                    .addCell(new HtmlTable.Cell("Disk"))
+                    .addCell(new HtmlTable.Cell("Memory")));
         }
 
         private void renderNodesOneType(
@@ -164,6 +175,7 @@ public class VdsClusterHtmlRenderer {
                 final ClusterStatsAggregator statsAggregator,
                 final double minMergeCompletionRatio,
                 final int maxPrematureCrashes,
+                final Map<String, Double> feedBlockLimits,
                 final EventLog eventLog,
                 final String pathPrefix,
                 final String dominantVtag,
@@ -188,6 +200,7 @@ public class VdsClusterHtmlRenderer {
                 addPrematureCrashes(maxPrematureCrashes, nodeInfo, row);
                 addEventsLastWeek(eventLog, currentTime, nodeInfo, row);
                 addBucketSpacesStats(nodeType, statsAggregator, minMergeCompletionRatio, nodeInfo, row);
+                addResourceUsage(nodeInfo, feedBlockLimits, row);
                 addStartTime(nodeInfo, row);
                 addRpcAddress(nodeInfo, row);
 
@@ -230,6 +243,37 @@ public class VdsClusterHtmlRenderer {
             }
         }
 
+        private void addResourceUsage(NodeInfo nodeInfo, Map<String, Double> feedBlockLimits, HtmlTable.Row row) {
+            if (nodeInfo.isDistributor()) {
+                row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY));
+                row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY));
+                return;
+            }
+            addSingleResourceUsageCell(nodeInfo, "disk", feedBlockLimits, row);
+            addSingleResourceUsageCell(nodeInfo, "memory", feedBlockLimits, row);
+        }
+
+        private void addSingleResourceUsageCell(NodeInfo nodeInfo, String resourceType,
+                                                Map<String, Double> feedBlockLimits, HtmlTable.Row row)
+        {
+            var hostInfo = nodeInfo.getHostInfo();
+            var usages = hostInfo.getContentNode().getResourceUsage();
+
+            var usage = usages.get(resourceType);
+            if (usage != null && usage.getUsage() != null) {
+                row.addCell(new HtmlTable.Cell(String.format("%.2f", usage.getUsage() * 100.0)));
+                double limit = feedBlockLimits.getOrDefault(resourceType, 1.0);
+                // Mark as error if limit exceeded, warn if within 5% of exceeding
+                if (usage.getUsage() > limit) {
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
+                } else if (usage.getUsage() > (limit - 0.05)) {
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
+                }
+            } else {
+                row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY));
+            }
+        }
+
         private void addEventsLastWeek(EventLog eventLog, long currentTime, NodeInfo nodeInfo, HtmlTable.Row row) {
             int nodeEvents = eventLog.getNodeEventsSince(nodeInfo.getNode(),
                     currentTime - eventLog.getRecentTimePeriod());
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java
index 11b07c037ae..3042e323879 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java
@@ -12,6 +12,7 @@ import org.junit.Test;
 
 import java.io.StringWriter;
 import java.io.Writer;
+import java.util.Collections;
 import java.util.TreeMap;
 
 import static org.hamcrest.core.Is.is;
@@ -60,6 +61,7 @@ public class ContentClusterHtmlRendrerTest {
                 statsAggregator,
                 1.0,
                 10,
+                Collections.emptyMap(),
                 eventLog,
                 "pathPrefix",
                 "name");