Add per node buckets pending stats on html status page.

6 files changed, 93 insertions, 34 deletions

diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
index 74ddd941afb..27612bb8b7d 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
@@ -46,6 +46,7 @@ public class ContentCluster {
             final StringBuilder sb,
             final Timer timer,
             final ClusterState state,
+            final ClusterStatsAggregator statsAggregator,
             final Distribution distribution,
             final FleetControllerOptions options,
             final EventLog eventLog) {
@@ -71,6 +72,7 @@ public class ContentCluster {
                     distributorNodeInfoByIndex,
                     timer,
                     state,
+                    statsAggregator,
                     options.maxPrematureCrashes,
                     eventLog,
                     clusterName,
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java
index 2698b079073..bc476923b23 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java
@@ -29,11 +29,11 @@ public class ContentClusterStats implements Iterable<ContentNodeStats> {
         return mapToNodeStats.values().iterator();
     }
 
-    ContentNodeStats getContentNode(Integer index) {
+    public ContentNodeStats getContentNode(Integer index) {
         return mapToNodeStats.get(index);
     }
 
-    int size() {
+    public int size() {
         return mapToNodeStats.size();
     }
 
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java
index 2063bdb24c9..1a57975c4bc 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java
@@ -78,6 +78,10 @@ public class StateVersionTracker {
         return currentClusterState.getBaselineClusterState();
     }
 
+    public ClusterStatsAggregator getAggregatedClusterStats() {
+        return clusterStateView.getStatsAggregator();
+    }
+
     public ClusterStateBundle getVersionedClusterStateBundle() {
         return currentClusterState;
     }
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
index 2ced8648ced..c7f83743708 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
@@ -65,6 +65,7 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa
                     content,
                     timer,
                     stateVersionTracker.getVersionedClusterState(),
+                    stateVersionTracker.getAggregatedClusterStats(),
                     data.getOptions().storageDistribution,
                     data.getOptions(),
                     eventLog
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java
index 2ed98855121..2544ac9e07c 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java
@@ -1,14 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core.status.statuspage;
 
+import com.yahoo.document.FixedBucketSpaces;
 import com.yahoo.vdslib.state.ClusterState;
 import com.yahoo.vdslib.state.NodeState;
 import com.yahoo.vdslib.state.NodeType;
 import com.yahoo.vdslib.state.State;
-import com.yahoo.vespa.clustercontroller.core.EventLog;
-import com.yahoo.vespa.clustercontroller.core.NodeInfo;
-import com.yahoo.vespa.clustercontroller.core.RealTimer;
-import com.yahoo.vespa.clustercontroller.core.Timer;
+import com.yahoo.vespa.clustercontroller.core.*;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -30,6 +28,9 @@ public class VdsClusterHtmlRendrer {
         private final HtmlTable.CellProperties headerProperties;
         private final StringBuilder contentBuilder = new StringBuilder();
         private final static String TAG_NOT_SET = "not set";
+        private final static HtmlTable.CellProperties WARNING_PROPERTY = new HtmlTable.CellProperties().setBackgroundColor(0xffffc0);
+        private final static HtmlTable.CellProperties ERROR_PROPERTY = new HtmlTable.CellProperties().setBackgroundColor(0xffc0c0);
+        private final static HtmlTable.CellProperties CENTERED_PROPERTY = new HtmlTable.CellProperties().align(HtmlTable.Orientation.CENTER);
 
         Table(final String clusterName, final int slobrokGenerationCount) {
             table.getTableProperties().align(HtmlTable.Orientation.RIGHT).setBackgroundColor(0xc0ffc0);
@@ -38,8 +39,8 @@ public class VdsClusterHtmlRendrer {
             table.getColProperties(2).align(HtmlTable.Orientation.LEFT);
             table.getColProperties(3).align(HtmlTable.Orientation.LEFT);
             table.getColProperties(7).align(HtmlTable.Orientation.LEFT);
-            table.getColProperties(12).align(HtmlTable.Orientation.LEFT);
-            for (int i = 4; i < 13; ++i) table.getColProperties(i).allowLineBreaks(false);
+            table.getColProperties(14).align(HtmlTable.Orientation.LEFT);
+            for (int i = 4; i < 15; ++i) table.getColProperties(i).allowLineBreaks(false);
             headerProperties = new HtmlTable.CellProperties()
                     .setBackgroundColor(0xffffff)
                     .align(HtmlTable.Orientation.CENTER);
@@ -64,6 +65,7 @@ public class VdsClusterHtmlRendrer {
                 final TreeMap<Integer, NodeInfo> distributorNodeInfos,
                 final Timer timer,
                 final ClusterState state,
+                final ClusterStatsAggregator statsAggregator,
                 final int maxPrematureCrashes,
                 final EventLog eventLog,
                 final String pathPrefix,
@@ -75,6 +77,7 @@ public class VdsClusterHtmlRendrer {
                     NodeType.STORAGE,
                     timer,
                     state,
+                    statsAggregator,
                     maxPrematureCrashes,
                     eventLog,
                     pathPrefix,
@@ -84,6 +87,7 @@ public class VdsClusterHtmlRendrer {
                     NodeType.DISTRIBUTOR,
                     timer,
                     state,
+                    statsAggregator,
                     maxPrematureCrashes,
                     eventLog,
                     pathPrefix,
@@ -134,12 +138,16 @@ public class VdsClusterHtmlRendrer {
                     .addCell(new HtmlTable.Cell("SSV<sup>4)</sup>"))
                     .addCell(new HtmlTable.Cell("PC<sup>5)</sup>"))
                     .addCell(new HtmlTable.Cell("ELW<sup>6)</sup>"))
+                    .addCell(new HtmlTable.Cell("Buckets pending")
+                            .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1)))
                     .addCell(new HtmlTable.Cell("Start Time"))
                     .addCell(new HtmlTable.Cell("RPC Address")));
             table.addRow(new HtmlTable.Row().setHeaderRow().addProperties(headerProperties)
                     .addCell(new HtmlTable.Cell("Reported"))
                     .addCell(new HtmlTable.Cell("Wanted"))
-                    .addCell(new HtmlTable.Cell("System")));
+                    .addCell(new HtmlTable.Cell("System"))
+                    .addCell(new HtmlTable.Cell(FixedBucketSpaces.defaultSpace()))
+                    .addCell(new HtmlTable.Cell(FixedBucketSpaces.globalSpace())));
         }
 
         private void renderNodesOneType(
@@ -147,6 +155,7 @@ public class VdsClusterHtmlRendrer {
                 final NodeType nodeType,
                 final Timer timer,
                 final ClusterState state,
+                final ClusterStatsAggregator statsAggregator,
                 final int maxPrematureCrashes,
                 final EventLog eventLog,
                 final String pathPrefix,
@@ -156,9 +165,6 @@ public class VdsClusterHtmlRendrer {
             addTableHeader(name, nodeType);
             for (final NodeInfo nodeInfo : nodeInfos.values()) {
                 HtmlTable.Row row = new HtmlTable.Row();
-                HtmlTable.CellProperties warning = new HtmlTable.CellProperties().setBackgroundColor(0xffffc0);
-                HtmlTable.CellProperties error = new HtmlTable.CellProperties().setBackgroundColor(0xffc0c0);
-                HtmlTable.CellProperties centered = new HtmlTable.CellProperties().align(HtmlTable.Orientation.CENTER);
 
                 // Add node index
                 row.addCell(new HtmlTable.Cell("<a href=\"" + pathPrefix + "/node=" + nodeInfo.getNode()
@@ -168,18 +174,18 @@ public class VdsClusterHtmlRendrer {
                 NodeState reportedState = nodeInfo.getReportedState().clone().setStartTimestamp(0);
                 row.addCell(new HtmlTable.Cell(HtmlTable.escape(reportedState.toString(true))));
                 if (!nodeInfo.getReportedState().getState().equals(State.UP)) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // Add wanted state
                 if (nodeInfo.getWantedState() == null || nodeInfo.getWantedState().getState().equals(State.UP)) {
-                    row.addCell(new HtmlTable.Cell("-").addProperties(centered));
+                    row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY));
                 } else {
                     row.addCell(new HtmlTable.Cell(HtmlTable.escape(nodeInfo.getWantedState().toString(true))));
                     if (nodeInfo.getWantedState().toString(true).indexOf("Disabled by fleet controller") != -1) {
-                        row.getLastCell().addProperties(error);
+                        row.getLastCell().addProperties(ERROR_PROPERTY);
                     } else {
-                        row.getLastCell().addProperties(warning);
+                        row.getLastCell().addProperties(WARNING_PROPERTY);
                     }
                 }
 
@@ -188,13 +194,13 @@ public class VdsClusterHtmlRendrer {
                 if (state.getClusterState().oneOf("uir")) {
                     row.addCell(new HtmlTable.Cell(HtmlTable.escape(ns.toString(true))));
                     if (ns.getState().equals(State.DOWN)) {
-                        row.getLastCell().addProperties(error);
+                        row.getLastCell().addProperties(ERROR_PROPERTY);
                     } else if (ns.getState().oneOf("mi")) {
-                        row.getLastCell().addProperties(warning);
+                        row.getLastCell().addProperties(WARNING_PROPERTY);
                     }
                 } else {
                     row.addCell(new HtmlTable.Cell("Cluster " +
-                            state.getClusterState().name().toLowerCase()).addProperties(error));
+                            state.getClusterState().name().toLowerCase()).addProperties(ERROR_PROPERTY));
                 }
 
                 // Add build tag version.
@@ -204,7 +210,7 @@ public class VdsClusterHtmlRendrer {
                                 : TAG_NOT_SET;
                 row.addCell(new HtmlTable.Cell(buildTagText));
                 if (! dominantVtag.equals(nodeInfo.getVtag())) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // Add failed connection attempt count
@@ -212,22 +218,22 @@ public class VdsClusterHtmlRendrer {
                 long timeSinceContact = nodeInfo.getTimeOfFirstFailingConnectionAttempt() == 0
                         ? 0 : currentTime - nodeInfo.getTimeOfFirstFailingConnectionAttempt();
                 if (timeSinceContact > 60 * 1000) {
-                    row.getLastCell().addProperties(error);
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
                 } else if (nodeInfo.getConnectionAttemptCount() > 0) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // Add time since first failing
                 row.addCell(new HtmlTable.Cell((timeSinceContact / 1000) + " s"));
                 if (timeSinceContact > 60 * 1000) {
-                    row.getLastCell().addProperties(error);
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
                 } else if (nodeInfo.getConnectionAttemptCount() > 0) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // State pending time
                 if (nodeInfo.getLatestNodeStateRequestTime() == null) {
-                    row.addCell(new HtmlTable.Cell("-").addProperties(centered));
+                    row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY));
                 } else {
                     row.addCell(new HtmlTable.Cell(HtmlTable.escape(RealTimer.printDuration(
                             currentTime - nodeInfo.getLatestNodeStateRequestTime()))));
@@ -236,17 +242,17 @@ public class VdsClusterHtmlRendrer {
                 // System state version
                 row.addCell(new HtmlTable.Cell("" + nodeInfo.getSystemStateVersionAcknowledged()));
                 if (nodeInfo.getSystemStateVersionAcknowledged() < state.getVersion() - 2) {
-                    row.getLastCell().addProperties(error);
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
                 } else if (nodeInfo.getSystemStateVersionAcknowledged() < state.getVersion()) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // Premature crashes
                 row.addCell(new HtmlTable.Cell("" + nodeInfo.getPrematureCrashCount()));
                 if (nodeInfo.getPrematureCrashCount() >= maxPrematureCrashes) {
-                    row.getLastCell().addProperties(error);
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
                 } else if (nodeInfo.getPrematureCrashCount() > 0) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // Events last week
@@ -254,14 +260,23 @@ public class VdsClusterHtmlRendrer {
                         currentTime - eventLog.getRecentTimePeriod());
                 row.addCell(new HtmlTable.Cell("" + nodeEvents));
                 if (nodeEvents > 20) {
-                    row.getLastCell().addProperties(error);
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
                 } else if (nodeEvents > 3) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
+                }
+
+                // Buckets pending ('default' and 'global' spaces)
+                if (nodeType.equals(NodeType.STORAGE)) {
+                    addBucketsPending(row, getStatsForContentNode(statsAggregator, nodeInfo, FixedBucketSpaces.defaultSpace()));
+                    addBucketsPending(row, getStatsForContentNode(statsAggregator, nodeInfo, FixedBucketSpaces.globalSpace()));
+                } else {
+                    addBucketsPending(row, getStatsForDistributorNode(statsAggregator, nodeInfo, FixedBucketSpaces.defaultSpace()));
+                    addBucketsPending(row, getStatsForDistributorNode(statsAggregator, nodeInfo, FixedBucketSpaces.globalSpace()));
                 }
 
                 // Start time
                 if (nodeInfo.getStartTimestamp() == 0) {
-                    row.addCell(new HtmlTable.Cell("-").addProperties(error).addProperties(centered));
+                    row.addCell(new HtmlTable.Cell("-").addProperties(ERROR_PROPERTY).addProperties(CENTERED_PROPERTY));
                 } else {
                     String startTime = RealTimer.printDateNoMilliSeconds(
                             1000 * nodeInfo.getStartTimestamp(), utcTimeZone);
@@ -270,16 +285,50 @@ public class VdsClusterHtmlRendrer {
 
                 // RPC address
                 if (nodeInfo.getRpcAddress() == null) {
-                    row.addCell(new HtmlTable.Cell("-").addProperties(error));
+                    row.addCell(new HtmlTable.Cell("-").addProperties(ERROR_PROPERTY));
                 } else {
                     row.addCell(new HtmlTable.Cell(HtmlTable.escape(nodeInfo.getRpcAddress())));
                     if (nodeInfo.isRpcAddressOutdated()) {
-                        row.getLastCell().addProperties(warning);
+                        row.getLastCell().addProperties(WARNING_PROPERTY);
                     }
                 }
                 table.addRow(row);
             }
         }
+
+        private static ContentNodeStats.BucketSpaceStats getStatsForContentNode(ClusterStatsAggregator statsAggregator,
+                                                                                NodeInfo nodeInfo,
+                                                                                String bucketSpace) {
+            ContentNodeStats nodeStats = statsAggregator.getAggregatedStats().getContentNode(nodeInfo.getNodeIndex());
+            if (nodeStats != null) {
+                return nodeStats.getBucketSpace(bucketSpace);
+            }
+            return null;
+        }
+
+        private static ContentNodeStats.BucketSpaceStats getStatsForDistributorNode(ClusterStatsAggregator statsAggregator,
+                                                                                    NodeInfo nodeInfo,
+                                                                                    String bucketSpace) {
+            ContentNodeStats nodeStats = statsAggregator.getAggregatedStatsForDistributor(nodeInfo.getNodeIndex());
+            return nodeStats.getBucketSpace(bucketSpace);
+        }
+
+        private static void addBucketsPending(HtmlTable.Row row, ContentNodeStats.BucketSpaceStats bucketSpaceStats) {
+            if (bucketSpaceStats != null) {
+                long bucketsPending = bucketSpaceStats.getBucketsPending();
+                String cellValue = "" + bucketsPending;
+                if (!bucketSpaceStats.valid()) {
+                    cellValue += "?";
+                }
+                row.addCell(new HtmlTable.Cell(cellValue));
+                if (bucketsPending > 0 || !bucketSpaceStats.valid()) {
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
+                }
+            } else {
+                row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY));
+            }
+        }
+
         private void addFooter(final StringBuilder contentBuilder, final long stableStateTimePeriode) {
             contentBuilder.append("<font size=\"-1\">\n")
                     .append("1) FC - Failed connections - We have tried to connect to the nodes this many times " +
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java
index 24d387f65de..6dc260d6bf3 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core;
 
+import com.google.common.collect.Sets;
 import com.yahoo.vdslib.state.ClusterState;
 import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo;
 import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRendrer;
@@ -48,12 +49,14 @@ public class ContentClusterHtmlRendrerTest {
             distributorNodeInfoByIndex.put(x, nodeInfo);
         }
         storageNodeInfoByIndex.put(2, new StorageNodeInfo(contentCluster, 2, false, "storage" + 2, null));
+        ClusterStatsAggregator statsAggregator = new ClusterStatsAggregator(Sets.newHashSet(2), Sets.newHashSet(2));
 
         table.renderNodes(
                 storageNodeInfoByIndex,
                 distributorNodeInfoByIndex,
                 new FakeTimer(),
                 state,
+                statsAggregator,
                 10,
                 eventLog,
                 "pathPrefix",