Merge pull request #16926 from vespa-engine/vekterli/dont-store-full-bundle-objects-in-state-history

Don't store full bundle objects in state history

8 files changed, 180 insertions, 119 deletions

diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateHistory.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateHistory.java
new file mode 100644
index 00000000000..ef747b71a26
--- /dev/null
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateHistory.java
@@ -0,0 +1,45 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.core;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Tracks a configurable max number of cluster state history entries, pruning old entries
+ * as newer entries are added. To save memory, state diffs are computed once and cached as
+ * a string upon adding a new state bundle. Only the most recent bundle is retained in its
+ * entirety.
+ */
+public class ClusterStateHistory {
+
+    private final LinkedList<ClusterStateHistoryEntry> stateHistory = new LinkedList<>();
+    private int maxHistoryEntryCount = 50;
+    private ClusterStateBundle prevStateBundle = null;
+
+    /**
+     * Sets limit on how many cluster states can be kept in the in-memory queue. Once
+     * the list exceeds this limit, the oldest state is repeatedly removed until the limit
+     * is no longer exceeded.
+     */
+    void setMaxHistoryEntryCount(final int maxHistoryEntryCount) {
+        this.maxHistoryEntryCount = maxHistoryEntryCount;
+    }
+
+    List<ClusterStateHistoryEntry> getHistory() {
+        return Collections.unmodifiableList(stateHistory);
+    }
+
+    public void add(ClusterStateBundle currentClusterState, long currentTimeMs) {
+        if (prevStateBundle != null) {
+            stateHistory.addFirst(ClusterStateHistoryEntry.makeSuccessor(currentClusterState, prevStateBundle, currentTimeMs));
+        } else {
+            stateHistory.addFirst(ClusterStateHistoryEntry.makeFirstEntry(currentClusterState, currentTimeMs));
+        }
+        prevStateBundle = currentClusterState;
+        while (stateHistory.size() > maxHistoryEntryCount) {
+            stateHistory.removeLast();
+        }
+    }
+
+}
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateHistoryEntry.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateHistoryEntry.java
index 65c0f210fd5..17c6d6b721f 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateHistoryEntry.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateHistoryEntry.java
@@ -5,25 +5,74 @@ import com.yahoo.vdslib.state.ClusterState;
 
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 import java.util.stream.Collectors;
 
+/**
+ * Represents the absolute cluster states (baseline + derived) for a given version
+ * as well as the relative diffs (baseline + derived) since the previous version.
+ */
 public class ClusterStateHistoryEntry {
 
-    private final ClusterStateBundle state;
+    public final static String BASELINE = "-";
+
     private final long time;
+    private final Map<String, String> states = new TreeMap<>();
+    private final Map<String, String> diffs = new TreeMap<>();
+
+    ClusterStateHistoryEntry(ClusterStateBundle state, long time) {
+        this.time = time;
+        populateStateStrings(state);
+        // No diffs for the first entry in the history.
+    }
 
-    ClusterStateHistoryEntry(final ClusterStateBundle state, final long time) {
-        this.state = state;
+    ClusterStateHistoryEntry(ClusterStateBundle state, ClusterStateBundle prevState, long time) {
         this.time = time;
+        populateStateStrings(state);
+        populateDiffStrings(state, prevState);
+    }
+
+    private void populateStateStrings(ClusterStateBundle state) {
+        states.put(BASELINE, state.getBaselineClusterState().toString());
+        var derivedStates = state.getDerivedBucketSpaceStates().entrySet().stream()
+                .collect(Collectors.toMap(
+                        entry -> entry.getKey(),
+                        entry -> entry.getValue().getClusterState().toString()));
+        states.putAll(derivedStates);
+    }
+
+    private void populateDiffStrings(ClusterStateBundle state, ClusterStateBundle prevState) {
+        // Yes, it's correct to get the diff by doing old.getHtmlDifference(new) rather than the other way around.
+        diffs.put(BASELINE, prevState.getBaselineClusterState().getHtmlDifference(state.getBaselineClusterState()));
+        var spaceDiffs = state.getDerivedBucketSpaceStates().entrySet().stream()
+                .collect(Collectors.toMap(
+                        entry -> entry.getKey(),
+                        entry -> derivedStateOf(prevState, entry.getKey()).getHtmlDifference(entry.getValue().getClusterState())));
+        diffs.putAll(spaceDiffs);
+    }
+
+    public static ClusterStateHistoryEntry makeFirstEntry(ClusterStateBundle state, long time) {
+        return new ClusterStateHistoryEntry(state, time);
+    }
+
+    public static ClusterStateHistoryEntry makeSuccessor(ClusterStateBundle state, ClusterStateBundle prevState, long time) {
+        return new ClusterStateHistoryEntry(state, prevState, time);
+    }
+
+    private static ClusterState derivedStateOf(ClusterStateBundle state, String space) {
+        return state.getDerivedBucketSpaceStates().getOrDefault(space, AnnotatedClusterState.emptyState()).getClusterState();
+    }
+
+    public Map<String, String> getRawStates() {
+        return states;
     }
 
-    public ClusterState getBaselineState() {
-        return state.getBaselineClusterState();
+    public String getStateString(String space) {
+        return states.getOrDefault(space, "");
     }
 
-    public Map<String, ClusterState> getDerivedBucketSpaceStates() {
-        return state.getDerivedBucketSpaceStates().entrySet().stream()
-                .collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().getClusterState()));
+    public String getDiffString(String space) {
+        return diffs.getOrDefault(space, "");
     }
 
     public long time() {
@@ -36,18 +85,19 @@ public class ClusterStateHistoryEntry {
         if (o == null || getClass() != o.getClass()) return false;
         ClusterStateHistoryEntry that = (ClusterStateHistoryEntry) o;
         return time == that.time &&
-                Objects.equals(state, that.state);
+                Objects.equals(states, that.states) &&
+                Objects.equals(diffs, that.diffs);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(state, time);
+        return Objects.hash(time, states, diffs);
     }
 
     // String representation only used for test expectation failures and debugging output.
     // Actual status page history entry rendering emits formatted date/time.
     public String toString() {
-        return String.format("state '%s' at time %d", state, time);
+        return String.format("state '%s' at time %d", getStateString(BASELINE), time);
     }
 
 }
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java
index 3c9b457e477..aa6e2973014 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java
@@ -4,8 +4,6 @@ package com.yahoo.vespa.clustercontroller.core;
 import com.yahoo.vdslib.state.ClusterState;
 import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo;
 
-import java.util.Collections;
-import java.util.LinkedList;
 import java.util.List;
 
 /**
@@ -34,8 +32,7 @@ public class StateVersionTracker {
     private ClusterStateView clusterStateView;
     private final ClusterStatsChangeTracker clusterStatsChangeTracker;
 
-    private final LinkedList<ClusterStateHistoryEntry> clusterStateHistory = new LinkedList<>();
-    private int maxHistoryEntryCount = 50;
+    private final ClusterStateHistory clusterStateHistory = new ClusterStateHistory();
     private double minMergeCompletionRatio;
 
     StateVersionTracker(double minMergeCompletionRatio) {
@@ -72,8 +69,8 @@ public class StateVersionTracker {
      *
      * Takes effect upon the next invocation of promoteCandidateToVersionedState().
      */
-    void setMaxHistoryEntryCount(final int maxHistoryEntryCount) {
-        this.maxHistoryEntryCount = maxHistoryEntryCount;
+    void setMaxHistoryEntryCount(int maxHistoryEntryCount) {
+        this.clusterStateHistory.setMaxHistoryEntryCount(maxHistoryEntryCount);
     }
 
     void setMinMergeCompletionRatio(double minMergeCompletionRatio) {
@@ -129,7 +126,7 @@ public class StateVersionTracker {
     }
 
     public List<ClusterStateHistoryEntry> getClusterStateHistory() {
-        return Collections.unmodifiableList(clusterStateHistory);
+        return clusterStateHistory.getHistory();
     }
 
     boolean candidateChangedEnoughFromCurrentToWarrantPublish() {
@@ -159,10 +156,7 @@ public class StateVersionTracker {
     }
 
     private void recordCurrentStateInHistoryAtTime(final long currentTimeMs) {
-        clusterStateHistory.addFirst(new ClusterStateHistoryEntry(currentClusterState, currentTimeMs));
-        while (clusterStateHistory.size() > maxHistoryEntryCount) {
-            clusterStateHistory.removeLast();
-        }
+        clusterStateHistory.add(currentClusterState, currentTimeMs);
     }
 
     void handleUpdatedHostInfo(final NodeInfo node, final HostInfo hostInfo) {
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java
index 5a1399c8f29..0364d46f582 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java
@@ -26,6 +26,7 @@ public class SystemStateBroadcaster {
     private final static long minTimeBetweenNodeErrorLogging = 10 * 60 * 1000;
     private final Map<Node, Long> lastErrorReported = new TreeMap<>();
 
+    private int lastOfficialStateVersion = 0;
     private int lastStateVersionBundleAcked = 0;
     private int lastClusterStateVersionConverged = 0;
     private ClusterStateBundle lastClusterStateBundleConverged;
@@ -255,6 +256,14 @@ public class SystemStateBroadcaster {
         return lastClusterStateVersionConverged == clusterStateBundle.getVersion();
     }
 
+    private boolean currentBundleVersionIsTaggedOfficial() {
+        return clusterStateBundle.getVersion() == lastOfficialStateVersion;
+    }
+
+    private void tagCurrentBundleVersionAsOfficial() {
+        lastOfficialStateVersion = clusterStateBundle.getVersion();
+    }
+
     public boolean broadcastNewStateBundleIfRequired(DatabaseHandler.Context dbContext, Communicator communicator,
                                                      int lastClusterStateVersionWrittenToZooKeeper) {
         if (clusterStateBundle == null) {
@@ -266,9 +275,9 @@ public class SystemStateBroadcaster {
 
         ClusterState baselineState = clusterStateBundle.getBaselineClusterState();
 
-        if (!baselineState.isOfficial()) {
+        if (!currentBundleVersionIsTaggedOfficial()) {
             log.log(Level.INFO, String.format("Publishing cluster state version %d", baselineState.getVersion()));
-            baselineState.setOfficial(true); // FIXME this violates state bundle immutability
+            tagCurrentBundleVersionAsOfficial();
         }
 
         List<NodeInfo> recipients = resolveStateVersionSendSet(dbContext);
@@ -291,7 +300,7 @@ public class SystemStateBroadcaster {
     }
 
     public boolean broadcastStateActivationsIfRequired(DatabaseHandler.Context dbContext, Communicator communicator) {
-        if (clusterStateBundle == null || !clusterStateBundle.getBaselineClusterState().isOfficial()) {
+        if (clusterStateBundle == null || !currentBundleVersionIsTaggedOfficial()) {
             return false;
         }
 
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
index 9d5e5e46f08..3a3503feefa 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
@@ -57,8 +57,6 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa
         content.append("<table><tr><td>UTC time when creating this page:</td><td align=\"right\">").append(RealTimer.printDateNoMilliSeconds(currentTime, tz)).append("</td></tr>");
         //content.append("<tr><td>Fleetcontroller version:</td><td align=\"right\">" + Vtag.V_TAG_PKG + "</td></tr/>");
         content.append("<tr><td>Cluster controller uptime:</td><td align=\"right\">" + RealTimer.printDuration(currentTime - startedTime) + "</td></tr></table>");
-        // State of master election
-        masterElectionHandler.writeHtmlState(content, data.getOptions().stateGatherCount);
         if (masterElectionHandler.isAmongNthFirst(data.getOptions().stateGatherCount)) {
             // Table overview of all the nodes
             cluster.writeHtmlState(
@@ -71,14 +69,16 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa
                     data.getOptions(),
                     eventLog
             );
-            // Overview of current config
-            data.getOptions().writeHtmlState(content);
             // Current cluster state and cluster state history
             writeHtmlState(stateVersionTracker, content, request);
         } else {
             // Overview of current config
             data.getOptions().writeHtmlState(content);
         }
+        // State of master election
+        masterElectionHandler.writeHtmlState(content, data.getOptions().stateGatherCount);
+        // Overview of current config
+        data.getOptions().writeHtmlState(content);
         // Event log
         eventLog.writeHtmlState(content, null);
         response.writeHtmlFooter(content, "");
@@ -101,25 +101,15 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa
         if ( ! stateVersionTracker.getClusterStateHistory().isEmpty()) {
             TimeZone tz = TimeZone.getTimeZone("UTC");
             sb.append("<h3 id=\"clusterstatehistory\">Cluster state history</h3>\n");
-            if (showLocal) {
-                sb.append("<p>Cluster states shown in gray are just transition states on the fleet controller and has never been sent to any nodes.</p>");
-            }
             sb.append("<table border=\"1\" cellspacing=\"0\"><tr>\n")
               .append("  <th>Creation date (").append(tz.getDisplayName(false, TimeZone.SHORT)).append(")</th>\n")
               .append("  <th>Bucket space</th>\n")
               .append("  <th>Cluster state</th>\n")
               .append("</tr>\n");
-            // Write cluster state history in reverse order (newest on top)
-            Iterator<ClusterStateHistoryEntry> stateIterator = stateVersionTracker.getClusterStateHistory().iterator();
-            ClusterStateHistoryEntry current = null;
-            while (stateIterator.hasNext()) {
-                ClusterStateHistoryEntry nextEntry = stateIterator.next();
-                if (nextEntry.getBaselineState().isOfficial() || showLocal) {
-                    if (current != null) writeClusterStateEntry(current, nextEntry, sb, tz);
-                    current = nextEntry;
-                }
+            // Write cluster state history in descending time point order (newest on top)
+            for (var historyEntry : stateVersionTracker.getClusterStateHistory()) {
+                writeClusterStateEntry(historyEntry, sb, tz);
             }
-            if (current != null) writeClusterStateEntry(current, null, sb, tz);
             sb.append("</table>\n");
         }
     }
@@ -131,59 +121,36 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa
         });
     }
 
-    private void writeClusterStateEntry(ClusterStateHistoryEntry entry, ClusterStateHistoryEntry last, StringBuilder sb, TimeZone tz) {
-        List<ClusterStateTransition> derivedTransitions = calculateDerivedClusterStateTransitions(entry, last);
-        sb.append("<tr><td rowspan=\"" + (derivedTransitions.size() + 1) + "\">").append(RealTimer.printDate(entry.time(), tz)).append("</td>");
-        writeClusterStateTransition(ClusterStateTransition.forBaseline(entry.getBaselineState(),
-                (last != null ? last.getBaselineState() : null)), entry.getBaselineState().isOfficial(), sb);
-        derivedTransitions.forEach(transition -> {
-            sb.append("<tr>");
-            writeClusterStateTransition(transition, entry.getBaselineState().isOfficial(), sb);
-        });
-    }
+    private void writeClusterStateEntry(ClusterStateHistoryEntry entry, StringBuilder sb, TimeZone tz) {
+        sb.append("<tr><td rowspan=\"").append(entry.getRawStates().size()).append("\">")
+          .append(RealTimer.printDate(entry.time(), tz)).append("</td>");
 
-    private void writeClusterStateTransition(ClusterStateTransition transition, boolean isOfficial, StringBuilder sb) {
-        sb.append("<td align=\"center\">").append(transition.bucketSpace).append("</td>");
-        sb.append("<td>").append(isOfficial ? "" : "<font color=\"grey\">");
-        sb.append(transition.current);
-        if (transition.last != null) {
-            sb.append("<br><b>Diff</b>: ").append(transition.last.getHtmlDifference(transition.current));
+        for (var space : entry.getRawStates().keySet()) {
+            if (!space.equals(ClusterStateHistoryEntry.BASELINE)) { // Always ordered first
+                sb.append("<tr>");
+            }
+            writeClusterStateTransition(space, entry.getStateString(space), entry.getDiffString(space),
+                                        entry.getStateString(ClusterStateHistoryEntry.BASELINE),
+                                        entry.getDiffString(ClusterStateHistoryEntry.BASELINE), sb);
         }
-        sb.append(isOfficial ? "" : "</font>").append("</td></tr>\n");
     }
 
-    private List<ClusterStateTransition> calculateDerivedClusterStateTransitions(ClusterStateHistoryEntry currentEntry,
-                                                                                 ClusterStateHistoryEntry lastEntry) {
-        List<ClusterStateTransition> result = new ArrayList<>();
-        currentEntry.getDerivedBucketSpaceStates().forEach((bucketSpace, currentState) -> {
-            ClusterState lastState = (lastEntry != null ? lastEntry.getDerivedBucketSpaceStates().get(bucketSpace) : null);
-            if ((!currentState.equals(currentEntry.getBaselineState())) ||
-                    ((lastState != null) && (!lastState.equals(lastEntry.getBaselineState())))) {
-                result.add(ClusterStateTransition.forBucketSpace(currentState, lastState, bucketSpace));
+    private void writeClusterStateTransition(String bucketSpace, String state, String diff,
+                                             String baselineState, String baselineDiff, StringBuilder sb) {
+        sb.append("<td align=\"center\">").append(bucketSpace).append("</td><td>");
+        if (!bucketSpace.equals(ClusterStateHistoryEntry.BASELINE) &&
+            state.equals(baselineState) && diff.equals(baselineDiff))
+        {
+            // Don't bother duplicating output for non-baseline states if they exactly match
+            // what's being output for the baseline state.
+            sb.append("<span style=\"color: gray\">(identical to baseline state)</span>");
+        } else {
+            sb.append(state);
+            if (!diff.isEmpty()) {
+                sb.append("<br><b>Diff</b>: ").append(diff);
             }
-        });
-        return result;
-    }
-
-    private static class ClusterStateTransition {
-        public final ClusterState current;
-        public final ClusterState last;
-        public final String bucketSpace;
-
-        private ClusterStateTransition(ClusterState current, ClusterState last, String bucketSpace) {
-            this.current = current;
-            this.last = last;
-            this.bucketSpace = bucketSpace;
         }
-
-        public static ClusterStateTransition forBaseline(ClusterState current, ClusterState last) {
-            return new ClusterStateTransition(current, last, "-");
-        }
-
-        public static ClusterStateTransition forBucketSpace(ClusterState current, ClusterState last, String bucketSpace) {
-            return new ClusterStateTransition(current, last, bucketSpace);
-        }
-
+        sb.append("</td></tr>\n");
     }
 
 }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateVersionTrackerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateVersionTrackerTest.java
index 83bdbfa0213..496d2a3b29a 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateVersionTrackerTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateVersionTrackerTest.java
@@ -180,8 +180,13 @@ public class StateVersionTrackerTest {
         assertTrue(versionTracker.getClusterStateHistory().isEmpty());
     }
 
-    private static ClusterStateHistoryEntry historyEntry(final String state, final long time) {
-        return new ClusterStateHistoryEntry(stateBundleWithoutAnnotations(state), time);
+    private static ClusterStateHistoryEntry historyEntry(String state, long time) {
+        return ClusterStateHistoryEntry.makeFirstEntry(stateBundleWithoutAnnotations(state), time);
+    }
+
+    private static ClusterStateHistoryEntry historyEntry(String state, String prevState, long time) {
+        return ClusterStateHistoryEntry.makeSuccessor(stateBundleWithoutAnnotations(state),
+                                                      stateBundleWithoutAnnotations(prevState), time);
     }
 
     @Test
@@ -191,12 +196,16 @@ public class StateVersionTrackerTest {
         updateAndPromote(versionTracker, stateWithoutAnnotations("distributor:3 storage:3"), 200);
         updateAndPromote(versionTracker, stateWithoutAnnotations("distributor:4 storage:4"), 300);
 
+        String s4 = "version:4 distributor:4 storage:4";
+        String s3 = "version:3 distributor:3 storage:3";
+        String s2 = "version:2 distributor:2 storage:2";
+
         // Note: newest entry first
         assertThat(versionTracker.getClusterStateHistory(),
                 equalTo(Arrays.asList(
-                         historyEntry("version:4 distributor:4 storage:4", 300),
-                         historyEntry("version:3 distributor:3 storage:3", 200),
-                         historyEntry("version:2 distributor:2 storage:2", 100))));
+                         historyEntry(s4, s3, 300),
+                         historyEntry(s3, s2, 200),
+                         historyEntry(s2, 100))));
     }
 
     @Test
@@ -208,17 +217,22 @@ public class StateVersionTrackerTest {
         updateAndPromote(versionTracker, stateWithoutAnnotations("distributor:3 storage:3"), 200);
         updateAndPromote(versionTracker, stateWithoutAnnotations("distributor:4 storage:4"), 300);
 
+        String s5 = "version:5 distributor:5 storage:5";
+        String s4 = "version:4 distributor:4 storage:4";
+        String s3 = "version:3 distributor:3 storage:3";
+        String s2 = "version:2 distributor:2 storage:2";
+
         assertThat(versionTracker.getClusterStateHistory(),
                 equalTo(Arrays.asList(
-                        historyEntry("version:4 distributor:4 storage:4", 300),
-                        historyEntry("version:3 distributor:3 storage:3", 200))));
+                        historyEntry(s4, s3, 300),
+                        historyEntry(s3, s2, 200))));
 
         updateAndPromote(versionTracker, stateWithoutAnnotations("distributor:5 storage:5"), 400);
 
         assertThat(versionTracker.getClusterStateHistory(),
                 equalTo(Arrays.asList(
-                        historyEntry("version:5 distributor:5 storage:5", 400),
-                        historyEntry("version:4 distributor:4 storage:4", 300))));
+                        historyEntry(s5, s4, 400),
+                        historyEntry(s4, s3, 300))));
     }
 
     @Test
diff --git a/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java b/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java
index b13b6230d3e..c536fc0f4cd 100644
--- a/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java
+++ b/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java
@@ -30,7 +30,6 @@ public class ClusterState implements Cloneable {
 
     private String description = "";
     private int distributionBits = 16;
-    private boolean official = false;
 
     public ClusterState(String serialized) throws ParseException {
         nodeCount.add(0);
@@ -169,16 +168,6 @@ public class ClusterState implements Cloneable {
                 : DEFAULT_DISTRIBUTOR_UP_NODE_STATE;
     }
 
-    /**
-     * Fleet controller marks states that are actually sent out to nodes as official states. Only fleetcontroller
-     * should set this to official, and only just before sending it out. This state is currently not serialized with
-     * the system state, but only used internally in the fleetcontroller. Might be useful client side though, where
-     * clients modify states to mark nodes down that they cannot speak with.
-     */
-    public void setOfficial(boolean official) { this.official = official; }
-    /** Whether this system state is an unmodified version of an official system state. */
-    public boolean isOfficial() { return official; }
-
     /** Used during deserialization */
     private class NodeData {
 
@@ -202,7 +191,6 @@ public class ClusterState implements Cloneable {
     }
 
     private void deserialize(String serialized) throws ParseException {
-        official = false;
         StringTokenizer st = new StringTokenizer(serialized, " \t\n\f\r", false);
         NodeData nodeData = new NodeData();
         String lastAbsolutePath = "";
@@ -323,9 +311,6 @@ public class ClusterState implements Cloneable {
         if (distributionBits != other.distributionBits) {
             diff.add(new Diff.Entry("bits", distributionBits, other.distributionBits));
         }
-        if (official != other.official) {
-            diff.add(new Diff.Entry("official", official, other.official));
-        }
         for (NodeType type : NodeType.getTypes()) {
             Diff typeDiff = new Diff();
             int maxCount = Math.max(getNodeCount(type), other.getNodeCount(type));
@@ -348,7 +333,6 @@ public class ClusterState implements Cloneable {
     }
 
     public void setVersion(int version) {
-        official = false;
         this.version = version;
     }
 
diff --git a/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java
index 258a79f5362..dbc8888cfda 100644
--- a/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java
+++ b/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java
@@ -188,9 +188,8 @@ public class ClusterStateTestCase{
         state2.setDistributionBits(21);
         state1.setVersion(123);
         state1.setNodeState(new Node(NodeType.STORAGE, 2), new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(0.2f).setDescription("Booting"));
-        state2.setOfficial(true);
 
-        assertEquals("version: 123 => 0, bits: 16 => 21, official: false => true, storage: [2: [Initializing => Up, description: Booting => ], 4: Down => Up, 5: Down => Up], distributor: [7: Up => Down, 8: Up => Down]", state1.getTextualDifference(state2));
+        assertEquals("version: 123 => 0, bits: 16 => 21, storage: [2: [Initializing => Up, description: Booting => ], 4: Down => Up, 5: Down => Up], distributor: [7: Up => Down, 8: Up => Down]", state1.getTextualDifference(state2));
     }
 
     @Test
@@ -221,9 +220,8 @@ public class ClusterStateTestCase{
         state1.setVersion(123);
         state1.setNodeState(new Node(NodeType.STORAGE, 2), new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(0.2f).setDescription("Booting"));
         state2.setDistributionBits(21);
-        state2.setOfficial(true);
-        assertEquals("version: 123 => 0, bits: 16 => 21, official: false => true, storage: [2: [Initializing => Up, description: Booting => ], 4: Down => Up, 5: Down => Up], distributor: [7: Up => Down, 8: Up => Down]", state1.getTextualDifference(state2));
-        assertEquals("version: 123 =&gt; 0, bits: 16 =&gt; 21, official: false =&gt; true, storage: [<br>\n" +
+        assertEquals("version: 123 => 0, bits: 16 => 21, storage: [2: [Initializing => Up, description: Booting => ], 4: Down => Up, 5: Down => Up], distributor: [7: Up => Down, 8: Up => Down]", state1.getTextualDifference(state2));
+        assertEquals("version: 123 =&gt; 0, bits: 16 =&gt; 21, storage: [<br>\n" +
                 "&nbsp;2: [<b>Initializing</b> =&gt; <b>Up</b>, description: Booting =&gt; ], <br>\n" +
                 "&nbsp;4: <b>Down</b> =&gt; <b>Up</b>, <br>\n" +
                 "&nbsp;5: <b>Down</b> =&gt; <b>Up</b><br>\n" +