// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.status; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.distribution.Group; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; import com.yahoo.vespa.clustercontroller.core.ClusterStateHistoryEntry; import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.EventLog; import com.yahoo.vespa.clustercontroller.core.FleetControllerOptions; import com.yahoo.vespa.clustercontroller.core.LeafGroups; import com.yahoo.vespa.clustercontroller.core.MasterElectionHandler; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.RealTimer; import com.yahoo.vespa.clustercontroller.core.StateVersionTracker; import com.yahoo.vespa.clustercontroller.core.Timer; import com.yahoo.vespa.clustercontroller.core.status.statuspage.HtmlTable; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageResponse; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRenderer; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.TimeZone; import java.util.TreeMap; import java.util.stream.Collectors; /** * @author Haakon Humberset */ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHandler { private static final DecimalFormat DecimalDot2 = new DecimalFormat("0.00", new DecimalFormatSymbols(Locale.ENGLISH)); private final Timer timer; private final ContentCluster cluster; private final MasterElectionHandler masterElectionHandler; private final StateVersionTracker stateVersionTracker; private final EventLog eventLog; private final long startedTime; private FleetControllerOptions options; public LegacyIndexPageRequestHandler(Timer timer, ContentCluster cluster, MasterElectionHandler masterElectionHandler, StateVersionTracker stateVersionTracker, EventLog eventLog, FleetControllerOptions options) { this.timer = timer; this.cluster = cluster; this.masterElectionHandler = masterElectionHandler; this.stateVersionTracker = stateVersionTracker; this.eventLog = eventLog; this.startedTime = timer.getCurrentTimeInMillis(); this.options = options; } public void propagateOptions(FleetControllerOptions options) { this.options = options; } @Override public StatusPageResponse handle(StatusPageServer.HttpRequest request) { TimeZone tz = TimeZone.getTimeZone("UTC"); long currentTime = timer.getCurrentTimeInMillis(); StatusPageResponse response = new StatusPageResponse(); response.setContentType("text/html"); StringBuilder content = new StringBuilder(); content.append("\n"); response.writeHtmlHeader(content, cluster.getName() + " Cluster Controller " + options.fleetControllerIndex() + " Status Page"); content.append("

") .append(" [ Current config") .append(" | Cluster states") .append(" | Event log") .append(" ]

\n"); content.append(""); content.append("
UTC time when creating this page:").append(RealTimer.printDateNoMilliSeconds(currentTime, tz)).append("
Cluster controller uptime:" + RealTimer.printDuration(currentTime - startedTime) + "
"); if (masterElectionHandler.isFirstInLine()) { // Table overview of all the nodes writeHtmlState(cluster, content, timer, stateVersionTracker, options, eventLog); // Current cluster state and cluster state history writeHtmlState(stateVersionTracker, content); } else { // Overview of current config writeHtmlState(content, options); } // State of master election masterElectionHandler.writeHtmlState(content); // Overview of current config writeHtmlState(content, options); // Event log eventLog.writeHtmlState(content, null); response.writeHtmlFooter(content, ""); response.writeContent(content.toString()); return response; } @Override public String pattern() { return "^/$"; } public void writeHtmlState(StateVersionTracker stateVersionTracker, StringBuilder sb) { sb.append("

Cluster states

\n"); writeClusterStates(sb, stateVersionTracker.getVersionedClusterStateBundle()); if ( ! stateVersionTracker.getClusterStateHistory().isEmpty()) { TimeZone tz = TimeZone.getTimeZone("UTC"); sb.append("

Cluster state history

\n"); sb.append("\n") .append(" \n") .append(" \n") .append(" \n") .append("\n"); // Write cluster state history in descending time point order (newest on top) for (var historyEntry : stateVersionTracker.getClusterStateHistory()) { writeClusterStateEntry(historyEntry, sb, tz); } sb.append("
Creation date (").append(tz.getDisplayName(false, TimeZone.SHORT)).append(")Bucket spaceCluster state
\n"); } } private static void writeClusterStates(StringBuilder sb, ClusterStateBundle clusterStates) { sb.append("

Baseline cluster state:
").append(clusterStates.getBaselineClusterState().toString()).append("

\n"); clusterStates.getDerivedBucketSpaceStates().forEach((bucketSpace, state) -> { sb.append("

" + bucketSpace + " cluster state:
").append(state.getClusterState().toString()).append("

\n"); }); } private void writeClusterStateEntry(ClusterStateHistoryEntry entry, StringBuilder sb, TimeZone tz) { sb.append("") .append(RealTimer.printDate(entry.time(), tz)).append(""); for (var space : entry.getRawStates().keySet()) { if (!space.equals(ClusterStateHistoryEntry.BASELINE)) { // Always ordered first sb.append(""); } writeClusterStateTransition(space, entry.getStateString(space), entry.getDiffString(space), entry.getStateString(ClusterStateHistoryEntry.BASELINE), entry.getDiffString(ClusterStateHistoryEntry.BASELINE), sb); } } private void writeClusterStateTransition(String bucketSpace, String state, String diff, String baselineState, String baselineDiff, StringBuilder sb) { sb.append("").append(bucketSpace).append(""); if (!bucketSpace.equals(ClusterStateHistoryEntry.BASELINE) && state.equals(baselineState) && diff.equals(baselineDiff)) { // Don't bother duplicating output for non-baseline states if they exactly match // what's being output for the baseline state. sb.append("(identical to baseline state)"); } else { sb.append(state); if (!diff.isEmpty()) { sb.append("
Diff: ").append(diff); } } sb.append("\n"); } private void writeHtmlState(ContentCluster cluster, StringBuilder sb, Timer timer, StateVersionTracker stateVersionTracker, FleetControllerOptions options, EventLog eventLog) { VdsClusterHtmlRenderer renderer = new VdsClusterHtmlRenderer(); VdsClusterHtmlRenderer.Table table = renderer.createNewClusterHtmlTable(cluster.getName(), cluster.getSlobrokGenerationCount()); ClusterStateBundle state = stateVersionTracker.getVersionedClusterStateBundle(); if (state.clusterFeedIsBlocked()) { // Implies FeedBlock != null table.appendRaw("

Cluster feeding is blocked!

\n"); table.appendRaw(String.format("

Summary: %s

\n", HtmlTable.escape(state.getFeedBlockOrNull().getDescription()))); } List groups = LeafGroups.enumerateFrom(cluster.getDistribution().getRootGroup()); for (Group group : groups) { assert (group != null); String localName = group.getUnixStylePath(); assert (localName != null); TreeMap storageNodeInfoByIndex = new TreeMap<>(); TreeMap distributorNodeInfoByIndex = new TreeMap<>(); for (ConfiguredNode configuredNode : group.getNodes()) { storeNodeInfo(cluster, configuredNode.index(), NodeType.STORAGE, storageNodeInfoByIndex); storeNodeInfo(cluster, configuredNode.index(), NodeType.DISTRIBUTOR, distributorNodeInfoByIndex); } table.renderNodes(storageNodeInfoByIndex, distributorNodeInfoByIndex, timer, state, stateVersionTracker.getAggregatedClusterStats(), options.minMergeCompletionRatio(), options.maxPrematureCrashes(), options.clusterFeedBlockLimit(), eventLog, cluster.getName(), localName); } table.addTable(sb, options.stableStateTimePeriod()); } private void storeNodeInfo(ContentCluster cluster, int nodeIndex, NodeType nodeType, Map nodeInfoByIndex) { NodeInfo nodeInfo = cluster.getNodeInfo(new Node(nodeType, nodeIndex)); if (nodeInfo == null) return; nodeInfoByIndex.put(nodeIndex, nodeInfo); } public void writeHtmlState(StringBuilder sb, FleetControllerOptions options) { String slobrokspecs = ""; for (int i = 0; i < options.slobrokConnectionSpecs().length; ++i) { if (i != 0) slobrokspecs += "
"; slobrokspecs += options.slobrokConnectionSpecs()[i]; } sb.append("

Current config

\n") .append("\n"); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); String zooKeeperAddress = splitZooKeeperAddress(options.zooKeeperServerAddress()); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append(""); sb.append("
PropertyValue
Cluster name").append(options.clusterName()).append("
Fleet controller index").append(options.fleetControllerIndex()).append("/").append(options.fleetControllerCount()).append("
Slobrok connection spec").append(slobrokspecs).append("
RPC port").append(options.rpcPort() == 0 ? "Pick random available" : options.rpcPort()).append("
HTTP port").append(options.httpPort() == 0 ? "Pick random available" : options.httpPort()).append("
Master cooldown period").append(RealTimer.printDuration(options.masterZooKeeperCooldownPeriod())).append("
Zookeeper server address").append(zooKeeperAddress).append("
Zookeeper session timeout").append(RealTimer.printDuration(options.zooKeeperSessionTimeout())).append("
Cycle wait time").append(options.cycleWaitTime()).append(" ms
Minimum time before first clusterstate broadcast as master").append(RealTimer.printDuration(options.minTimeBeforeFirstSystemStateBroadcast())).append("
Minimum time between official cluster states").append(RealTimer.printDuration(options.minTimeBetweenNewSystemStates())).append("
Node state request timeout").append(RealTimer.printDuration(options.nodeStateRequestTimeoutMS())).append("
Maximum distributor transition time").append(RealTimer.printDuration(options.maxTransitionTime().get(NodeType.DISTRIBUTOR))).append("
Maximum storage transition time").append(RealTimer.printDuration(options.maxTransitionTime().get(NodeType.STORAGE))).append("
Maximum initialize without progress time").append(RealTimer.printDuration(options.maxInitProgressTime())).append("
Maximum premature crashes").append(options.maxPrematureCrashes()).append("
Stable state time period").append(RealTimer.printDuration(options.stableStateTimePeriod())).append("
Slobrok disconnect grace period").append(RealTimer.printDuration(options.maxSlobrokDisconnectGracePeriod())).append("
Number of distributor nodes").append(options.nodes() == null ? "Autodetect" : options.nodes().size()).append("
Number of storage nodes").append(options.nodes() == null ? "Autodetect" : options.nodes().size()).append("
Minimum distributor nodes being up for cluster to be up").append(options.minDistributorNodesUp()).append("
Minimum storage nodes being up for cluster to be up").append(options.minStorageNodesUp()).append("
Minimum percentage of distributor nodes being up for cluster to be up").append(DecimalDot2.format(100 * options.minRatioOfDistributorNodesUp())).append(" %
Minimum percentage of storage nodes being up for cluster to be up").append(DecimalDot2.format(100 * options.minRatioOfStorageNodesUp())).append(" %
Show local cluster state changes").append(options.showLocalSystemStatesInEventLog()).append("
Maximum event log size").append(options.eventLogMaxSize()).append("
Maximum node event log size").append(options.eventNodeLogMaxSize()).append("
Wanted distribution bits").append(options.distributionBits()).append("
Max deferred task version wait time").append(options.maxDeferredTaskVersionWaitTime().toMillis()).append("ms
Cluster has global document types configured").append(options.clusterHasGlobalDocumentTypes()).append("
Enable 2-phase cluster state activation protocol").append(options.enableTwoPhaseClusterStateActivation()).append("
Cluster auto feed block on resource exhaustion enabled") .append(options.clusterFeedBlockEnabled()).append("
Feed block limits") .append(options.clusterFeedBlockLimit().entrySet().stream() .map(kv -> String.format("%s: %.2f%%", kv.getKey(), kv.getValue() * 100.0)) .sorted() .collect(Collectors.joining("
"))).append("
"); } private static String splitZooKeeperAddress(String s) { StringBuilder sb = new StringBuilder(); while (true) { int index = s.indexOf(','); if (index > 0) { sb.append(s.substring(0, index + 1)).append(' '); s = s.substring(index+1); } else { break; } } sb.append(s); return sb.toString(); } }