diff options
Diffstat (limited to 'controller-server/src/main/java/com')
6 files changed, 121 insertions, 129 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 9c414ce8348..d70c65f343a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import com.yahoo.vespa.hosted.controller.deployment.JobList; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -20,6 +21,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -38,18 +40,21 @@ public class MetricsReporter extends Maintainer { public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades"; public static final String DEPLOYMENT_BUILD_AGE_SECONDS = "deployment.buildAgeSeconds"; public static final String DEPLOYMENT_WARNINGS = "deployment.warnings"; - public static final String NODES_FAILING_SYSTEM_UPGRADE = "deployment.nodesFailingSystemUpgrade"; + // TODO(mpolden): Remove these two metrics + public static final String NODES_FAILING_PLATFORM_UPGRADE = "deployment.nodesFailingSystemUpgrade"; public static final String NODES_FAILING_OS_UPGRADE = "deployment.nodesFailingOsUpgrade"; + public static final String OS_CHANGE_DURATION = "deployment.osChangeDuration"; + public static final String PLATFORM_CHANGE_DURATION = "deployment.platformChangeDuration"; public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; - // The time a node belonging to a system application can spend from being told to upgrade until the upgrade is - // completed. Nodes exceeding this time are counted as failures. - private static final Duration NODE_UPGRADE_TIMEOUT = Duration.ofMinutes(90); + // The time a system application node can spend after suspending for Vespa upgrade until the upgrade is completed. + // Nodes exceeding this budget are counted as failures. + private static final Duration PLATFORM_UPGRADE_BUDGET = Duration.ofMinutes(30); - // The time a single node can spend performing an OS upgrade after being told to upgrade. Nodes exceeding this time - // multiplied by the number of nodes upgrading are counted as failures. - private static final Duration OS_UPGRADE_TIME_ALLOWANCE_PER_NODE = Duration.ofMinutes(30); + // The time a system application node can spend after suspending foor OS upgrade until the upgrade is completed. + // Nodes exceeding this budget are counted as failures. + private static final Duration OS_UPGRADE_BUDGET = Duration.ofMinutes(30); private final Metric metric; private final Clock clock; @@ -65,7 +70,7 @@ public class MetricsReporter extends Maintainer { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); - reportNodesFailingUpgrade(); + reportChangeDurations(); } private void reportRemainingRotations() { @@ -107,43 +112,44 @@ public class MetricsReporter extends Maintainer { metric.createContext(Map.of())); } - private void reportNodesFailingUpgrade() { - metric.set(NODES_FAILING_SYSTEM_UPGRADE, nodesFailingSystemUpgrade(), metric.createContext(Map.of())); - metric.set(NODES_FAILING_OS_UPGRADE, nodesFailingOsUpgrade(), metric.createContext(Map.of())); + private void reportChangeDurations() { + var platformChangeDurations = platformChangeDurations(); + var osChangeDurations = osChangeDurations(); + var nodesFailingSystemUpgrade = platformChangeDurations.values().stream() + .filter(duration -> duration.compareTo(PLATFORM_UPGRADE_BUDGET) > 0) + .count(); + var nodesFailingOsUpgrade = osChangeDurations.values().stream() + .filter(duration -> duration.compareTo(OS_UPGRADE_BUDGET) > 0) + .count(); + metric.set(NODES_FAILING_PLATFORM_UPGRADE, nodesFailingSystemUpgrade, metric.createContext(Map.of())); + metric.set(NODES_FAILING_OS_UPGRADE, nodesFailingOsUpgrade, metric.createContext(Map.of())); + platformChangeDurations.forEach((nodeVersion, duration) -> { + metric.set(PLATFORM_CHANGE_DURATION, duration.toSeconds(), metric.createContext(dimensions(nodeVersion))); + }); + osChangeDurations.forEach((nodeVersion, duration) -> { + metric.set(OS_CHANGE_DURATION, duration.toSeconds(), metric.createContext(dimensions(nodeVersion))); + }); } - private int nodesFailingSystemUpgrade() { - if (!controller().versionStatus().isUpgrading()) return 0; - return nodesFailingUpgrade(controller().versionStatus().versions(), (vespaVersion) -> { - if (vespaVersion.confidence() == VespaVersion.Confidence.broken) return NodeVersions.EMPTY; - return vespaVersion.nodeVersions(); - }, NODE_UPGRADE_TIMEOUT); + private Map<NodeVersion, Duration> platformChangeDurations() { + return changeDurations(controller().versionStatus().versions(), VespaVersion::nodeVersions); } - private int nodesFailingOsUpgrade() { - var allNodeVersions = controller().osVersionStatus().versions().values(); - var totalTimeout = 0L; - for (var nodeVersions : allNodeVersions) { - for (var nodeVersion : nodeVersions.asMap().values()) { - if (!nodeVersion.changing()) continue; - totalTimeout += OS_UPGRADE_TIME_ALLOWANCE_PER_NODE.toMillis(); - } - } - return nodesFailingUpgrade(allNodeVersions, Function.identity(), Duration.ofMillis(totalTimeout)); + private Map<NodeVersion, Duration> osChangeDurations() { + return changeDurations(controller().osVersionStatus().versions().values(), Function.identity()); } - private <V> int nodesFailingUpgrade(Collection<V> collection, Function<V, NodeVersions> nodeVersionsFunction, Duration timeout) { - var nodesFailingUpgrade = 0; - var acceptableInstant = clock.instant().minus(timeout); - for (var object : collection) { - for (var nodeVersion : nodeVersionsFunction.apply(object).asMap().values()) { - if (!nodeVersion.changing()) continue; - if (nodeVersion.changedAt().isBefore(acceptableInstant)) nodesFailingUpgrade++; + private <V> Map<NodeVersion, Duration> changeDurations(Collection<V> versions, Function<V, NodeVersions> versionsGetter) { + var now = clock.instant(); + var durations = new HashMap<NodeVersion, Duration>(); + for (var version : versions) { + for (var nodeVersion : versionsGetter.apply(version).asMap().values()) { + durations.put(nodeVersion, nodeVersion.changeDuration(now)); } } - return nodesFailingUpgrade; + return durations; } - + private static double deploymentFailRatio(DeploymentStatusList statuses) { return statuses.asList().stream() .mapToInt(status -> status.hasFailures() ? 1 : 0) @@ -198,6 +204,11 @@ public class MetricsReporter extends Maintainer { "app",application.application().value() + "." + application.instance().value()); } + private static Map<String, String> dimensions(NodeVersion nodeVersion) { + return Map.of("host", nodeVersion.hostname().value(), + "zone", nodeVersion.zone().value()); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java index 6e9def40e44..f9f8de96591 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java @@ -11,8 +11,6 @@ import com.yahoo.slime.Inspector; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; -import java.time.Instant; - /** * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.NodeVersion}. * @@ -30,7 +28,7 @@ public class NodeVersionSerializer { private static final String hostnameField = "hostname"; private static final String zoneField = "zone"; private static final String wantedVersionField = "wantedVersion"; - private static final String changedAtField = "changedAt"; + private static final String suspendedAtField = "suspendedAt"; public void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) { for (var nodeVersion : nodeVersions.asMap().values()) { @@ -38,7 +36,8 @@ public class NodeVersionSerializer { nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value()); nodeVersionObject.setString(zoneField, nodeVersion.zone().value()); nodeVersionObject.setString(wantedVersionField, nodeVersion.wantedVersion().toFullString()); - nodeVersionObject.setLong(changedAtField, nodeVersion.changedAt().toEpochMilli()); + nodeVersion.suspendedAt().ifPresent(suspendedAt -> nodeVersionObject.setLong(suspendedAtField, + suspendedAt.toEpochMilli())); } } @@ -48,8 +47,8 @@ public class NodeVersionSerializer { var hostname = HostName.from(entry.field(hostnameField).asString()); var zone = ZoneId.from(entry.field(zoneField).asString()); var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString()); - var changedAt = Instant.ofEpochMilli(entry.field(changedAtField).asLong()); - nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, changedAt)); + var suspendedAt = Serializers.optionalInstant(entry.field(suspendedAtField)); + nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt)); }); return new NodeVersions(nodeVersions.build()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java index d82cddb4779..4a00add411f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java @@ -5,8 +5,10 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; +import java.time.Duration; import java.time.Instant; import java.util.Objects; +import java.util.Optional; /** * Version information for a node allocated to a {@link com.yahoo.vespa.hosted.controller.application.SystemApplication}. @@ -21,14 +23,15 @@ public class NodeVersion { private final ZoneId zone; private final Version currentVersion; private final Version wantedVersion; - private final Instant changedAt; + private final Optional<Instant> suspendedAt; - public NodeVersion(HostName hostname, ZoneId zone, Version currentVersion, Version wantedVersion, Instant changedAt) { + public NodeVersion(HostName hostname, ZoneId zone, Version currentVersion, Version wantedVersion, + Optional<Instant> suspendedAt) { this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); this.zone = Objects.requireNonNull(zone, "zone must be non-null"); this.currentVersion = Objects.requireNonNull(currentVersion, "version must be non-null"); this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null"); - this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); + this.suspendedAt = Objects.requireNonNull(suspendedAt, "suspendedAt must be non-null"); } /** Hostname of this */ @@ -51,31 +54,39 @@ public class NodeVersion { return wantedVersion; } - /** Returns whether this is changing (upgrading or downgrading) */ - public boolean changing() { - return !currentVersion.equals(wantedVersion); + /** Returns the duration of the change in this, measured relative to instant */ + public Duration changeDuration(Instant instant) { + if (!changing()) return Duration.ZERO; + if (suspendedAt.isEmpty()) return Duration.ZERO; // Node hasn't suspended to apply the change yet + return Duration.between(suspendedAt.get(), instant).abs(); } - /** The most recent time the version of this changed */ - public Instant changedAt() { - return changedAt; + /** The most recent time the node referenced by this suspended. This is empty if the node is not suspended. */ + public Optional<Instant> suspendedAt() { + return suspendedAt; } /** Returns a copy of this with current version set to given version */ - public NodeVersion withCurrentVersion(Version version, Instant changedAt) { + public NodeVersion withCurrentVersion(Version version) { if (currentVersion.equals(version)) return this; - return new NodeVersion(hostname, zone, version, wantedVersion, changedAt); + return new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt); } /** Returns a copy of this with wanted version set to given version */ public NodeVersion withWantedVersion(Version version) { if (wantedVersion.equals(version)) return this; - return new NodeVersion(hostname, zone, currentVersion, version, changedAt); + return new NodeVersion(hostname, zone, currentVersion, version, suspendedAt); + } + + /** Returns a copy of this with wanted version set to given version */ + public NodeVersion withSuspendedAt(Optional<Instant> suspendedAt) { + if (suspendedAt.equals(this.suspendedAt)) return this; + return new NodeVersion(hostname, zone, currentVersion, wantedVersion, suspendedAt); } @Override public String toString() { - return hostname + ": " + currentVersion + " -> " + wantedVersion + " [zone=" + zone + ", changedAt=" + changedAt + "]"; + return hostname + ": " + currentVersion + " -> " + wantedVersion + " [zone=" + zone + ", suspendedAt=" + suspendedAt.map(Instant::toString).orElse("<not suspended>") + "]"; } @Override @@ -87,12 +98,17 @@ public class NodeVersion { zone.equals(that.zone) && currentVersion.equals(that.currentVersion) && wantedVersion.equals(that.wantedVersion) && - changedAt.equals(that.changedAt); + suspendedAt.equals(that.suspendedAt); } @Override public int hashCode() { - return Objects.hash(hostname, zone, currentVersion, wantedVersion, changedAt); + return Objects.hash(hostname, zone, currentVersion, wantedVersion, suspendedAt); + } + + /** Returns whether this is changing (upgrading or downgrading) */ + private boolean changing() { + return !currentVersion.equals(wantedVersion); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java index 3ab96e03bcd..4ce0a35e96f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java @@ -7,6 +7,7 @@ import com.google.common.collect.ListMultimap; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -20,12 +21,10 @@ import java.util.function.Predicate; */ public class NodeVersions { - public static final NodeVersions EMPTY = new NodeVersions(ImmutableMap.of()); - private final ImmutableMap<HostName, NodeVersion> nodeVersions; - public NodeVersions(ImmutableMap<HostName, NodeVersion> nodeVersions) { - this.nodeVersions = Objects.requireNonNull(nodeVersions); + public NodeVersions(Map<HostName, NodeVersion> nodeVersions) { + this.nodeVersions = ImmutableMap.copyOf(Objects.requireNonNull(nodeVersions)); } public Map<HostName, NodeVersion> asMap() { @@ -48,7 +47,7 @@ public class NodeVersions { /** Returns a copy of this containing only node versions of given version */ public NodeVersions matching(Version version) { - return filter(nodeVersion -> nodeVersion.currentVersion().equals(version)); + return copyOf(nodeVersions.values(), nodeVersion -> nodeVersion.currentVersion().equals(version)); } /** Returns number of node versions in this */ @@ -56,31 +55,6 @@ public class NodeVersions { return nodeVersions.size(); } - /** Returns a copy of this containing only the given node versions */ - public NodeVersions with(List<NodeVersion> nodeVersions) { - var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); - for (var nodeVersion : nodeVersions) { - var existing = this.nodeVersions.get(nodeVersion.hostname()); - if (existing != null) { - newNodeVersions.put(nodeVersion.hostname(), existing.withCurrentVersion(nodeVersion.currentVersion(), - nodeVersion.changedAt()) - .withWantedVersion(nodeVersion.wantedVersion())); - } else { - newNodeVersions.put(nodeVersion.hostname(), nodeVersion); - } - } - return new NodeVersions(newNodeVersions.build()); - } - - private NodeVersions filter(Predicate<NodeVersion> predicate) { - var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); - for (var kv : nodeVersions.entrySet()) { - if (!predicate.test(kv.getValue())) continue; - newNodeVersions.put(kv.getKey(), kv.getValue()); - } - return new NodeVersions(newNodeVersions.build()); - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -94,4 +68,21 @@ public class NodeVersions { return Objects.hash(nodeVersions); } + public static NodeVersions copyOf(List<NodeVersion> nodeVersions) { + return copyOf(nodeVersions, (ignored) -> true); + } + + public static NodeVersions copyOf(Map<HostName, NodeVersion> nodeVersions) { + return new NodeVersions(nodeVersions); + } + + private static NodeVersions copyOf(Collection<NodeVersion> nodeVersions, Predicate<NodeVersion> predicate) { + var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); + for (var nodeVersion : nodeVersions) { + if (!predicate.test(nodeVersion)) continue; + newNodeVersions.put(nodeVersion.hostname(), nodeVersion); + } + return new NodeVersions(newNodeVersions.build()); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 1773a9c122e..1dffd1383bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -16,7 +16,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -59,9 +58,7 @@ public class OsVersionStatus { /** Compute the current OS versions in this system. This is expensive and should be called infrequently */ public static OsVersionStatus compute(Controller controller) { - var osVersionStatus = controller.osVersionStatus(); var osVersions = new HashMap<OsVersion, List<NodeVersion>>(); - var now = controller.clock().instant(); controller.osVersions().forEach(osVersion -> osVersions.put(osVersion, new ArrayList<>())); for (var application : SystemApplication.all()) { @@ -71,19 +68,16 @@ public class OsVersionStatus { .targetVersionsOf(zone.getId()) .osVersion(application.nodeType()) .orElse(Version.emptyVersion); - controller.serviceRegistry().configServer().nodeRepository() - .list(zone.getId(), application.id()).stream() - .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) - .map(node -> new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), targetOsVersion, now)) - .forEach(nodeVersion -> { - var newNodeVersion = osVersionStatus.of(nodeVersion.hostname()) - .map(nv -> nv.withCurrentVersion(nodeVersion.currentVersion(), now) - .withWantedVersion(nodeVersion.wantedVersion())) - .orElse(nodeVersion); - var version = new OsVersion(newNodeVersion.currentVersion(), zone.getCloudName()); - osVersions.putIfAbsent(version, new ArrayList<>()); - osVersions.get(version).add(newNodeVersion); - }); + + for (var node : controller.serviceRegistry().configServer().nodeRepository().list(zone.getId(), application.id())) { + if (!OsUpgrader.eligibleForUpgrade(node, application)) continue; + var suspendedAt = node.suspendedSince(); + var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), + targetOsVersion, suspendedAt); + var osVersion = new OsVersion(nodeVersion.currentVersion(), zone.getCloudName()); + osVersions.putIfAbsent(osVersion, new ArrayList<>()); + osVersions.get(osVersion).add(nodeVersion); + } } } @@ -98,15 +92,6 @@ public class OsVersionStatus { return new OsVersionStatus(newOsVersions.build()); } - /** Returns version of node identified by given host name */ - private Optional<NodeVersion> of(HostName hostname) { - return versions.values().stream() - .map(nodeVersions -> nodeVersions.asMap().get(hostname)) - .map(Optional::ofNullable) - .flatMap(Optional::stream) - .findFirst(); - } - private static List<ZoneApi> zonesToUpgrade(Controller controller) { return controller.zoneRegistry().osUpgradePolicies().stream() .flatMap(upgradePolicy -> upgradePolicy.asList().stream()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 0868d7ca695..7618e8210e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -8,23 +8,15 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; -import com.yahoo.vespa.hosted.controller.deployment.JobStatus; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -149,10 +141,7 @@ public class VersionStatus { } private static NodeVersions findSystemApplicationVersions(Controller controller) { - var nodeVersions = controller.versionStatus().systemVersion() - .map(VespaVersion::nodeVersions) - .orElse(NodeVersions.EMPTY); - var newNodeVersions = new ArrayList<NodeVersion>(); + var nodeVersions = new LinkedHashMap<HostName, NodeVersion>(); for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) { for (var application : SystemApplication.all()) { var nodes = controller.serviceRegistry().configServer().nodeRepository() @@ -165,15 +154,16 @@ public class VersionStatus { log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() + " has not converged"); } - var now = controller.clock().instant(); for (var node : nodes) { // Only use current node version if config has converged - Version version = configConverged ? node.currentVersion() : controller.systemVersion(); - newNodeVersions.add(new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(), now)); + var version = configConverged ? node.currentVersion() : controller.systemVersion(); + var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(), + node.suspendedSince()); + nodeVersions.put(nodeVersion.hostname(), nodeVersion); } } } - return nodeVersions.with(newNodeVersions); + return NodeVersions.copyOf(nodeVersions); } private static ListMultimap<ControllerVersion, HostName> findControllerVersions(Controller controller) { |