Merge pull request #5177 from vespa-engine/geirst/add-buckets-pending-stats-on-clustercontroller-status-page

Geirst/add buckets pending stats on clustercontroller status page

10 files changed, 187 insertions, 55 deletions

diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java
index 3b8cb14e977..ed2ca32dab0 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java
@@ -27,6 +27,7 @@ import java.util.*;
 public class ClusterStatsAggregator {
 
     private final Set<Integer> distributors;
+    private final Set<Integer> storageNodes;
     private final Set<Integer> nonUpdatedDistributors;
 
     // Maps the distributor node index to a map of content node index to the
@@ -40,14 +41,26 @@ public class ClusterStatsAggregator {
 
     ClusterStatsAggregator(Set<Integer> distributors, Set<Integer> storageNodes) {
         this.distributors = distributors;
+        this.storageNodes = storageNodes;
         nonUpdatedDistributors = new HashSet<>(distributors);
         aggregatedStats = new ContentClusterStats(storageNodes);
     }
 
-    ContentClusterStats getAggregatedStats() {
+    public ContentClusterStats getAggregatedStats() {
         return aggregatedStats;
     }
 
+    public ContentNodeStats getAggregatedStatsForDistributor(int distributorIndex) {
+        ContentNodeStats result = new ContentNodeStats(distributorIndex);
+        ContentClusterStats distributorStats = distributorToStats.get(distributorIndex);
+        if (distributorStats != null) {
+            for (Iterator<ContentNodeStats> itr = distributorStats.iterator(); itr.hasNext(); ) {
+                result.add(itr.next());
+            }
+        }
+        return result;
+    }
+
     boolean hasUpdatesFromAllDistributors() {
         return nonUpdatedDistributors.isEmpty();
     }
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
index 74ddd941afb..27612bb8b7d 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
@@ -46,6 +46,7 @@ public class ContentCluster {
             final StringBuilder sb,
             final Timer timer,
             final ClusterState state,
+            final ClusterStatsAggregator statsAggregator,
             final Distribution distribution,
             final FleetControllerOptions options,
             final EventLog eventLog) {
@@ -71,6 +72,7 @@ public class ContentCluster {
                     distributorNodeInfoByIndex,
                     timer,
                     state,
+                    statsAggregator,
                     options.maxPrematureCrashes,
                     eventLog,
                     clusterName,
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java
index 2698b079073..bc476923b23 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java
@@ -29,11 +29,11 @@ public class ContentClusterStats implements Iterable<ContentNodeStats> {
         return mapToNodeStats.values().iterator();
     }
 
-    ContentNodeStats getContentNode(Integer index) {
+    public ContentNodeStats getContentNode(Integer index) {
         return mapToNodeStats.get(index);
     }
 
-    int size() {
+    public int size() {
         return mapToNodeStats.size();
     }
 
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java
index 2063bdb24c9..1a57975c4bc 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java
@@ -78,6 +78,10 @@ public class StateVersionTracker {
         return currentClusterState.getBaselineClusterState();
     }
 
+    public ClusterStatsAggregator getAggregatedClusterStats() {
+        return clusterStateView.getStatsAggregator();
+    }
+
     public ClusterStateBundle getVersionedClusterStateBundle() {
         return currentClusterState;
     }
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
index 2ced8648ced..c7f83743708 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
@@ -65,6 +65,7 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa
                     content,
                     timer,
                     stateVersionTracker.getVersionedClusterState(),
+                    stateVersionTracker.getAggregatedClusterStats(),
                     data.getOptions().storageDistribution,
                     data.getOptions(),
                     eventLog
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java
index 2ed98855121..eb377718614 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java
@@ -1,14 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core.status.statuspage;
 
+import com.yahoo.document.FixedBucketSpaces;
 import com.yahoo.vdslib.state.ClusterState;
 import com.yahoo.vdslib.state.NodeState;
 import com.yahoo.vdslib.state.NodeType;
 import com.yahoo.vdslib.state.State;
-import com.yahoo.vespa.clustercontroller.core.EventLog;
-import com.yahoo.vespa.clustercontroller.core.NodeInfo;
-import com.yahoo.vespa.clustercontroller.core.RealTimer;
-import com.yahoo.vespa.clustercontroller.core.Timer;
+import com.yahoo.vespa.clustercontroller.core.*;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -30,6 +28,9 @@ public class VdsClusterHtmlRendrer {
         private final HtmlTable.CellProperties headerProperties;
         private final StringBuilder contentBuilder = new StringBuilder();
         private final static String TAG_NOT_SET = "not set";
+        private final static HtmlTable.CellProperties WARNING_PROPERTY = new HtmlTable.CellProperties().setBackgroundColor(0xffffc0);
+        private final static HtmlTable.CellProperties ERROR_PROPERTY = new HtmlTable.CellProperties().setBackgroundColor(0xffc0c0);
+        private final static HtmlTable.CellProperties CENTERED_PROPERTY = new HtmlTable.CellProperties().align(HtmlTable.Orientation.CENTER);
 
         Table(final String clusterName, final int slobrokGenerationCount) {
             table.getTableProperties().align(HtmlTable.Orientation.RIGHT).setBackgroundColor(0xc0ffc0);
@@ -38,8 +39,8 @@ public class VdsClusterHtmlRendrer {
             table.getColProperties(2).align(HtmlTable.Orientation.LEFT);
             table.getColProperties(3).align(HtmlTable.Orientation.LEFT);
             table.getColProperties(7).align(HtmlTable.Orientation.LEFT);
-            table.getColProperties(12).align(HtmlTable.Orientation.LEFT);
-            for (int i = 4; i < 13; ++i) table.getColProperties(i).allowLineBreaks(false);
+            table.getColProperties(14).align(HtmlTable.Orientation.LEFT);
+            for (int i = 4; i < 15; ++i) table.getColProperties(i).allowLineBreaks(false);
             headerProperties = new HtmlTable.CellProperties()
                     .setBackgroundColor(0xffffff)
                     .align(HtmlTable.Orientation.CENTER);
@@ -64,6 +65,7 @@ public class VdsClusterHtmlRendrer {
                 final TreeMap<Integer, NodeInfo> distributorNodeInfos,
                 final Timer timer,
                 final ClusterState state,
+                final ClusterStatsAggregator statsAggregator,
                 final int maxPrematureCrashes,
                 final EventLog eventLog,
                 final String pathPrefix,
@@ -75,6 +77,7 @@ public class VdsClusterHtmlRendrer {
                     NodeType.STORAGE,
                     timer,
                     state,
+                    statsAggregator,
                     maxPrematureCrashes,
                     eventLog,
                     pathPrefix,
@@ -84,6 +87,7 @@ public class VdsClusterHtmlRendrer {
                     NodeType.DISTRIBUTOR,
                     timer,
                     state,
+                    statsAggregator,
                     maxPrematureCrashes,
                     eventLog,
                     pathPrefix,
@@ -134,12 +138,16 @@ public class VdsClusterHtmlRendrer {
                     .addCell(new HtmlTable.Cell("SSV<sup>4)</sup>"))
                     .addCell(new HtmlTable.Cell("PC<sup>5)</sup>"))
                     .addCell(new HtmlTable.Cell("ELW<sup>6)</sup>"))
+                    .addCell(new HtmlTable.Cell("Buckets pending")
+                            .addProperties(new HtmlTable.CellProperties().setColSpan(2).setRowSpan(1)))
                     .addCell(new HtmlTable.Cell("Start Time"))
                     .addCell(new HtmlTable.Cell("RPC Address")));
             table.addRow(new HtmlTable.Row().setHeaderRow().addProperties(headerProperties)
                     .addCell(new HtmlTable.Cell("Reported"))
                     .addCell(new HtmlTable.Cell("Wanted"))
-                    .addCell(new HtmlTable.Cell("System")));
+                    .addCell(new HtmlTable.Cell("System"))
+                    .addCell(new HtmlTable.Cell(FixedBucketSpaces.defaultSpace()))
+                    .addCell(new HtmlTable.Cell(FixedBucketSpaces.globalSpace())));
         }
 
         private void renderNodesOneType(
@@ -147,6 +155,7 @@ public class VdsClusterHtmlRendrer {
                 final NodeType nodeType,
                 final Timer timer,
                 final ClusterState state,
+                final ClusterStatsAggregator statsAggregator,
                 final int maxPrematureCrashes,
                 final EventLog eventLog,
                 final String pathPrefix,
@@ -156,9 +165,6 @@ public class VdsClusterHtmlRendrer {
             addTableHeader(name, nodeType);
             for (final NodeInfo nodeInfo : nodeInfos.values()) {
                 HtmlTable.Row row = new HtmlTable.Row();
-                HtmlTable.CellProperties warning = new HtmlTable.CellProperties().setBackgroundColor(0xffffc0);
-                HtmlTable.CellProperties error = new HtmlTable.CellProperties().setBackgroundColor(0xffc0c0);
-                HtmlTable.CellProperties centered = new HtmlTable.CellProperties().align(HtmlTable.Orientation.CENTER);
 
                 // Add node index
                 row.addCell(new HtmlTable.Cell("<a href=\"" + pathPrefix + "/node=" + nodeInfo.getNode()
@@ -168,18 +174,18 @@ public class VdsClusterHtmlRendrer {
                 NodeState reportedState = nodeInfo.getReportedState().clone().setStartTimestamp(0);
                 row.addCell(new HtmlTable.Cell(HtmlTable.escape(reportedState.toString(true))));
                 if (!nodeInfo.getReportedState().getState().equals(State.UP)) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // Add wanted state
                 if (nodeInfo.getWantedState() == null || nodeInfo.getWantedState().getState().equals(State.UP)) {
-                    row.addCell(new HtmlTable.Cell("-").addProperties(centered));
+                    row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY));
                 } else {
                     row.addCell(new HtmlTable.Cell(HtmlTable.escape(nodeInfo.getWantedState().toString(true))));
                     if (nodeInfo.getWantedState().toString(true).indexOf("Disabled by fleet controller") != -1) {
-                        row.getLastCell().addProperties(error);
+                        row.getLastCell().addProperties(ERROR_PROPERTY);
                     } else {
-                        row.getLastCell().addProperties(warning);
+                        row.getLastCell().addProperties(WARNING_PROPERTY);
                     }
                 }
 
@@ -188,13 +194,13 @@ public class VdsClusterHtmlRendrer {
                 if (state.getClusterState().oneOf("uir")) {
                     row.addCell(new HtmlTable.Cell(HtmlTable.escape(ns.toString(true))));
                     if (ns.getState().equals(State.DOWN)) {
-                        row.getLastCell().addProperties(error);
+                        row.getLastCell().addProperties(ERROR_PROPERTY);
                     } else if (ns.getState().oneOf("mi")) {
-                        row.getLastCell().addProperties(warning);
+                        row.getLastCell().addProperties(WARNING_PROPERTY);
                     }
                 } else {
                     row.addCell(new HtmlTable.Cell("Cluster " +
-                            state.getClusterState().name().toLowerCase()).addProperties(error));
+                            state.getClusterState().name().toLowerCase()).addProperties(ERROR_PROPERTY));
                 }
 
                 // Add build tag version.
@@ -204,7 +210,7 @@ public class VdsClusterHtmlRendrer {
                                 : TAG_NOT_SET;
                 row.addCell(new HtmlTable.Cell(buildTagText));
                 if (! dominantVtag.equals(nodeInfo.getVtag())) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // Add failed connection attempt count
@@ -212,22 +218,22 @@ public class VdsClusterHtmlRendrer {
                 long timeSinceContact = nodeInfo.getTimeOfFirstFailingConnectionAttempt() == 0
                         ? 0 : currentTime - nodeInfo.getTimeOfFirstFailingConnectionAttempt();
                 if (timeSinceContact > 60 * 1000) {
-                    row.getLastCell().addProperties(error);
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
                 } else if (nodeInfo.getConnectionAttemptCount() > 0) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // Add time since first failing
                 row.addCell(new HtmlTable.Cell((timeSinceContact / 1000) + " s"));
                 if (timeSinceContact > 60 * 1000) {
-                    row.getLastCell().addProperties(error);
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
                 } else if (nodeInfo.getConnectionAttemptCount() > 0) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // State pending time
                 if (nodeInfo.getLatestNodeStateRequestTime() == null) {
-                    row.addCell(new HtmlTable.Cell("-").addProperties(centered));
+                    row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY));
                 } else {
                     row.addCell(new HtmlTable.Cell(HtmlTable.escape(RealTimer.printDuration(
                             currentTime - nodeInfo.getLatestNodeStateRequestTime()))));
@@ -236,17 +242,17 @@ public class VdsClusterHtmlRendrer {
                 // System state version
                 row.addCell(new HtmlTable.Cell("" + nodeInfo.getSystemStateVersionAcknowledged()));
                 if (nodeInfo.getSystemStateVersionAcknowledged() < state.getVersion() - 2) {
-                    row.getLastCell().addProperties(error);
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
                 } else if (nodeInfo.getSystemStateVersionAcknowledged() < state.getVersion()) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // Premature crashes
                 row.addCell(new HtmlTable.Cell("" + nodeInfo.getPrematureCrashCount()));
                 if (nodeInfo.getPrematureCrashCount() >= maxPrematureCrashes) {
-                    row.getLastCell().addProperties(error);
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
                 } else if (nodeInfo.getPrematureCrashCount() > 0) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
                 }
 
                 // Events last week
@@ -254,14 +260,23 @@ public class VdsClusterHtmlRendrer {
                         currentTime - eventLog.getRecentTimePeriod());
                 row.addCell(new HtmlTable.Cell("" + nodeEvents));
                 if (nodeEvents > 20) {
-                    row.getLastCell().addProperties(error);
+                    row.getLastCell().addProperties(ERROR_PROPERTY);
                 } else if (nodeEvents > 3) {
-                    row.getLastCell().addProperties(warning);
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
+                }
+
+                // Buckets pending ('default' and 'global' spaces)
+                if (nodeType.equals(NodeType.STORAGE)) {
+                    addBucketsPending(row, getStatsForContentNode(statsAggregator, nodeInfo, FixedBucketSpaces.defaultSpace()));
+                    addBucketsPending(row, getStatsForContentNode(statsAggregator, nodeInfo, FixedBucketSpaces.globalSpace()));
+                } else {
+                    addBucketsPending(row, getStatsForDistributorNode(statsAggregator, nodeInfo, FixedBucketSpaces.defaultSpace()));
+                    addBucketsPending(row, getStatsForDistributorNode(statsAggregator, nodeInfo, FixedBucketSpaces.globalSpace()));
                 }
 
                 // Start time
                 if (nodeInfo.getStartTimestamp() == 0) {
-                    row.addCell(new HtmlTable.Cell("-").addProperties(error).addProperties(centered));
+                    row.addCell(new HtmlTable.Cell("-").addProperties(ERROR_PROPERTY).addProperties(CENTERED_PROPERTY));
                 } else {
                     String startTime = RealTimer.printDateNoMilliSeconds(
                             1000 * nodeInfo.getStartTimestamp(), utcTimeZone);
@@ -270,16 +285,50 @@ public class VdsClusterHtmlRendrer {
 
                 // RPC address
                 if (nodeInfo.getRpcAddress() == null) {
-                    row.addCell(new HtmlTable.Cell("-").addProperties(error));
+                    row.addCell(new HtmlTable.Cell("-").addProperties(ERROR_PROPERTY));
                 } else {
                     row.addCell(new HtmlTable.Cell(HtmlTable.escape(nodeInfo.getRpcAddress())));
                     if (nodeInfo.isRpcAddressOutdated()) {
-                        row.getLastCell().addProperties(warning);
+                        row.getLastCell().addProperties(WARNING_PROPERTY);
                     }
                 }
                 table.addRow(row);
             }
         }
+
+        private static ContentNodeStats.BucketSpaceStats getStatsForContentNode(ClusterStatsAggregator statsAggregator,
+                                                                                NodeInfo nodeInfo,
+                                                                                String bucketSpace) {
+            ContentNodeStats nodeStats = statsAggregator.getAggregatedStats().getContentNode(nodeInfo.getNodeIndex());
+            if (nodeStats != null) {
+                return nodeStats.getBucketSpace(bucketSpace);
+            }
+            return null;
+        }
+
+        private static ContentNodeStats.BucketSpaceStats getStatsForDistributorNode(ClusterStatsAggregator statsAggregator,
+                                                                                    NodeInfo nodeInfo,
+                                                                                    String bucketSpace) {
+            ContentNodeStats nodeStats = statsAggregator.getAggregatedStatsForDistributor(nodeInfo.getNodeIndex());
+            return nodeStats.getBucketSpace(bucketSpace);
+        }
+
+        private static void addBucketsPending(HtmlTable.Row row, ContentNodeStats.BucketSpaceStats bucketSpaceStats) {
+            if (bucketSpaceStats != null) {
+                long bucketsPending = bucketSpaceStats.getBucketsPending();
+                String cellValue = String.valueOf(bucketsPending);
+                if (!bucketSpaceStats.valid()) {
+                    cellValue += "?";
+                }
+                row.addCell(new HtmlTable.Cell(cellValue));
+                if (bucketsPending > 0 || !bucketSpaceStats.valid()) {
+                    row.getLastCell().addProperties(WARNING_PROPERTY);
+                }
+            } else {
+                row.addCell(new HtmlTable.Cell("-").addProperties(CENTERED_PROPERTY));
+            }
+        }
+
         private void addFooter(final StringBuilder contentBuilder, final long stableStateTimePeriode) {
             contentBuilder.append("<font size=\"-1\">\n")
                     .append("1) FC - Failed connections - We have tried to connect to the nodes this many times " +
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java
index ac6417d0077..90c9700e001 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java
@@ -18,24 +18,50 @@ public class ClusterStatsAggregatorTest {
 
     private static class Fixture {
         private ClusterStatsAggregator aggregator;
+
         public Fixture(Set<Integer> distributorNodes,
                 Set<Integer> contentNodes) {
             aggregator = new ClusterStatsAggregator(distributorNodes, contentNodes);
         }
+
         public void update(int distributorIndex, ContentClusterStatsBuilder clusterStats) {
             aggregator.updateForDistributor(distributorIndex, clusterStats.build());
         }
+
         public void verify(ContentClusterStatsBuilder expectedStats) {
             assertEquals(expectedStats.build(), aggregator.getAggregatedStats());
         }
+
+        public void verify(int distributorIndex, ContentNodeStatsBuilder expectedStats) {
+            assertEquals(expectedStats.build(), aggregator.getAggregatedStatsForDistributor(distributorIndex));
+        }
+
         public boolean hasUpdatesFromAllDistributors() {
             return aggregator.hasUpdatesFromAllDistributors();
         }
+
         public boolean mayHaveBucketsPendingInGlobalSpace() {
             return aggregator.mayHaveBucketsPendingInGlobalSpace();
         }
     }
 
+    private static class FourNodesFixture extends Fixture {
+        public FourNodesFixture() {
+            super(distributorNodes(1, 2), contentNodes(3, 4));
+
+            update(1, new ContentClusterStatsBuilder()
+                    .add(3, "default", 10, 1)
+                    .add(3, "global", 11, 2)
+                    .add(4, "default", 12, 3)
+                    .add(4, "global", 13, 4));
+            update(2, new ContentClusterStatsBuilder()
+                    .add(3, "default", 14, 5)
+                    .add(3, "global", 15, 6)
+                    .add(4, "default", 16, 7)
+                    .add(4, "global", 17, 8));
+        }
+    }
+
     private static Set<Integer> distributorNodes(Integer... indices) {
         return Sets.newHashSet(indices);
     }
@@ -56,18 +82,8 @@ public class ClusterStatsAggregatorTest {
 
     @Test
     public void aggregator_handles_updates_to_multiple_distributors_and_content_nodes() {
-        Fixture f = new Fixture(distributorNodes(1, 2), contentNodes(3, 4));
+        Fixture f = new FourNodesFixture();
 
-        f.update(1, new ContentClusterStatsBuilder()
-                .add(3, "default", 10, 1)
-                .add(3, "global", 11, 2)
-                .add(4, "default", 12, 3)
-                .add(4, "global", 13, 4));
-        f.update(2, new ContentClusterStatsBuilder()
-                .add(3, "default", 14, 5)
-                .add(3, "global", 15, 6)
-                .add(4, "default", 16, 7)
-                .add(4, "global", 17, 8));
         f.verify(new ContentClusterStatsBuilder()
                 .add(3, "default", 10 + 14, 1 + 5)
                 .add(3, "global", 11 + 15, 2 + 6)
@@ -154,4 +170,17 @@ public class ClusterStatsAggregatorTest {
         assertFalse(f.mayHaveBucketsPendingInGlobalSpace());
     }
 
+    @Test
+    public void aggregator_can_provide_aggregated_stats_per_distributor() {
+        Fixture f = new FourNodesFixture();
+
+        f.verify(1, ContentNodeStatsBuilder.forNode(1)
+                .add("default", 10 + 12, 1 + 3)
+                .add("global", 11 + 13, 2 + 4));
+
+        f.verify(2, ContentNodeStatsBuilder.forNode(2)
+                .add("default", 14 + 16, 5 + 7)
+                .add("global", 15 + 17, 6 + 8));
+    }
+
 }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java
index 24d387f65de..6dc260d6bf3 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core;
 
+import com.google.common.collect.Sets;
 import com.yahoo.vdslib.state.ClusterState;
 import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo;
 import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRendrer;
@@ -48,12 +49,14 @@ public class ContentClusterHtmlRendrerTest {
             distributorNodeInfoByIndex.put(x, nodeInfo);
         }
         storageNodeInfoByIndex.put(2, new StorageNodeInfo(contentCluster, 2, false, "storage" + 2, null));
+        ClusterStatsAggregator statsAggregator = new ClusterStatsAggregator(Sets.newHashSet(2), Sets.newHashSet(2));
 
         table.renderNodes(
                 storageNodeInfoByIndex,
                 distributorNodeInfoByIndex,
                 new FakeTimer(),
                 state,
+                statsAggregator,
                 10,
                 eventLog,
                 "pathPrefix",
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStatsBuilder.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStatsBuilder.java
index 16767cafa8f..dd0205658cb 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStatsBuilder.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStatsBuilder.java
@@ -9,7 +9,7 @@ import java.util.Map;
  */
 public class ContentClusterStatsBuilder {
 
-    private final Map<Integer, Map<String, ContentNodeStats.BucketSpaceStats>> stats = new HashMap<>();
+    private final Map<Integer, ContentNodeStatsBuilder> stats = new HashMap<>();
 
     public ContentClusterStatsBuilder add(int nodeIndex, String bucketSpace, long bucketsTotal, long bucketsPending) {
         return add(nodeIndex, bucketSpace, ContentNodeStats.BucketSpaceStats.of(bucketsTotal, bucketsPending));
@@ -24,24 +24,24 @@ public class ContentClusterStatsBuilder {
     }
 
     public ContentClusterStatsBuilder add(int nodeIndex, String bucketSpace, ContentNodeStats.BucketSpaceStats bucketSpaceStats) {
-        Map<String, ContentNodeStats.BucketSpaceStats> contentNodeStats = stats.get(nodeIndex);
-        if (contentNodeStats == null) {
-            contentNodeStats = new HashMap<>();
-            stats.put(nodeIndex, contentNodeStats);
+        ContentNodeStatsBuilder nodeStatsBuilder = stats.get(nodeIndex);
+        if (nodeStatsBuilder == null) {
+            nodeStatsBuilder = ContentNodeStatsBuilder.forNode(nodeIndex);
+            stats.put(nodeIndex, nodeStatsBuilder);
         }
-        contentNodeStats.put(bucketSpace, bucketSpaceStats);
+        nodeStatsBuilder.add(bucketSpace, bucketSpaceStats);
         return this;
     }
 
     public ContentClusterStatsBuilder add(int nodeIndex) {
-        stats.put(nodeIndex, new HashMap<>());
+        stats.put(nodeIndex, ContentNodeStatsBuilder.forNode(nodeIndex));
         return this;
     }
 
     public ContentClusterStats build() {
         Map<Integer, ContentNodeStats> nodeToStatsMap = new HashMap<>();
-        stats.forEach((nodeIndex, bucketSpaces) ->
-                nodeToStatsMap.put(nodeIndex, new ContentNodeStats(nodeIndex, bucketSpaces)));
+        stats.forEach((nodeIndex, nodeStatsBuilder) ->
+                nodeToStatsMap.put(nodeIndex, nodeStatsBuilder.build()));
         return new ContentClusterStats(nodeToStatsMap);
     }
 }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java
new file mode 100644
index 00000000000..9fb020260e9
--- /dev/null
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java
@@ -0,0 +1,31 @@
+package com.yahoo.vespa.clustercontroller.core;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ContentNodeStatsBuilder {
+
+    private final int nodeIndex;
+    private final Map<String, ContentNodeStats.BucketSpaceStats> stats = new HashMap<>();
+
+    private ContentNodeStatsBuilder(int nodeIndex) {
+        this.nodeIndex = nodeIndex;
+    }
+
+    public static ContentNodeStatsBuilder forNode(int nodeIndex) {
+        return new ContentNodeStatsBuilder(nodeIndex);
+    }
+
+    public ContentNodeStatsBuilder add(String bucketSpace, long bucketsTotal, long bucketsPending) {
+        return add(bucketSpace, ContentNodeStats.BucketSpaceStats.of(bucketsTotal, bucketsPending));
+    }
+
+    public ContentNodeStatsBuilder add(String bucketSpace, ContentNodeStats.BucketSpaceStats bucketSpaceStats) {
+        stats.put(bucketSpace, bucketSpaceStats);
+        return this;
+    }
+
+    public ContentNodeStats build() {
+        return new ContentNodeStats(nodeIndex, stats);
+    }
+}