From dd2739ec3abce6ca0ab3341b0ef6968f48f26492 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Mon, 7 Oct 2019 09:41:43 +0200 Subject: Replace OsVersion.Node with NodeVersion --- .../hosted/controller/maintenance/OsUpgrader.java | 2 +- .../hosted/controller/persistence/CuratorDb.java | 2 +- .../persistence/NodeVersionSerializer.java | 56 ++++++---- .../persistence/OsVersionStatusSerializer.java | 60 +++++----- .../persistence/VersionStatusSerializer.java | 15 +-- .../hosted/controller/restapi/os/OsApiHandler.java | 10 +- .../controller/versions/OsVersionStatus.java | 122 ++++++++------------- .../controller/maintenance/OsUpgraderTest.java | 8 +- .../maintenance/OsVersionStatusUpdaterTest.java | 11 +- .../persistence/OsVersionStatusSerializerTest.java | 56 +++++++--- .../testdata/os-version-status-legacy-format.json | 22 ++++ 11 files changed, 187 insertions(+), 177 deletions(-) create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/os-version-status-legacy-format.json (limited to 'controller-server/src') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 60bc3d15ec6..93d1dac7382 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -61,7 +61,7 @@ public class OsUpgrader extends InfrastructureUpgrader { // Return target if we have nodes in this cloud on a lower version return controller().osVersion(cloud) .filter(target -> controller().osVersionStatus().nodesIn(cloud).stream() - .anyMatch(node -> node.version().isBefore(target.version()))) + .anyMatch(node -> node.currentVersion().isBefore(target.version()))) .map(OsVersion::version); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index b22b8845c24..dbd52fc6d02 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -89,7 +89,7 @@ public class CuratorDb { private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); private final RunSerializer runSerializer = new RunSerializer(); private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); - private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer); + private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer(); private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java index d0e785198b1..4b6e997241d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; @@ -8,13 +9,9 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Optional; /** * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.NodeVersion}. @@ -32,37 +29,50 @@ public class NodeVersionSerializer { private static final String hostnameField = "hostname"; private static final String zoneField = "zone"; - private static final String currentVersionField = "currentVersion"; private static final String wantedVersionField = "wantedVersion"; private static final String changedAtField = "changedAt"; - public void nodeVersionsToSlime(Collection nodeVersions, Cursor array, boolean writeCurrentVersion) { - for (var nodeVersion : nodeVersions) { + // Legacy fields + private static final String environmentField = "environment"; + private static final String regionField = "region"; + + public void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) { + for (var nodeVersion : nodeVersions.asMap().values()) { var nodeVersionObject = array.addObject(); nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value()); nodeVersionObject.setString(zoneField, nodeVersion.zone().value()); - if (writeCurrentVersion) { - nodeVersionObject.setString(currentVersionField, nodeVersion.currentVersion().toFullString()); - } nodeVersionObject.setString(wantedVersionField, nodeVersion.wantedVersion().toFullString()); nodeVersionObject.setLong(changedAtField, nodeVersion.changedAt().toEpochMilli()); } } - public List nodeVersionsFromSlime(Inspector object, Optional version) { - var nodeVersions = new ArrayList(); - object.traverse((ArrayTraverser) (i, entry) -> { + public NodeVersions nodeVersionsFromSlime(Inspector array, Version version) { + var nodeVersions = ImmutableMap.builder(); + array.traverse((ArrayTraverser) (i, entry) -> { var hostname = HostName.from(entry.field(hostnameField).asString()); - // TODO(mpolden): Make non-optional after September 2019 - var zone = Serializers.optionalString(entry.field(zoneField)) - .map(ZoneId::from) - .orElseGet(ZoneId::defaultId); - var currentVersion = version.orElseGet(() -> Version.fromString(entry.field(currentVersionField).asString())); - var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString()); - var changedAt = Instant.ofEpochMilli(entry.field(changedAtField).asLong()); - nodeVersions.add(new NodeVersion(hostname, zone, currentVersion, wantedVersion, changedAt)); + var zone = zoneFromSlime(entry); + // TODO(mpolden): Make the following fields non-optional after September 2019 + var wantedVersion = Serializers.optionalString(entry.field(wantedVersionField)) + .map(Version::fromString) + .orElse(Version.emptyVersion); + var changedAt = Serializers.optionalInstant(entry.field(changedAtField)).orElse(Instant.EPOCH); + nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, changedAt)); }); - return Collections.unmodifiableList(nodeVersions); + return new NodeVersions(nodeVersions.build()); + } + + // TODO(mpolden): Simplify and in-line after September 2019 + private ZoneId zoneFromSlime(Inspector object) { + var zoneInspector = object.field(zoneField); + if (zoneInspector.valid()) { + return ZoneId.from(zoneInspector.asString()); + } + var regionInspector = object.field(regionField); + var environmentInspector = object.field(environmentField); + if (regionInspector.valid() && environmentInspector.valid()) { + return ZoneId.from(environmentInspector.asString(), regionInspector.asString()); + } + return ZoneId.defaultId(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java index 88805f54d65..fa29969f166 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java @@ -1,23 +1,19 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; import com.yahoo.component.Version; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.TreeMap; /** * Serializer for {@link OsVersionStatus}. @@ -39,11 +35,14 @@ public class OsVersionStatusSerializer { private static final String hostnameField = "hostname"; private static final String regionField = "region"; private static final String environmentField = "environment"; + private static final String nodeVersionsField = "nodeVersions"; private final OsVersionSerializer osVersionSerializer; + private final NodeVersionSerializer nodeVersionSerializer; - public OsVersionStatusSerializer(OsVersionSerializer osVersionSerializer) { + public OsVersionStatusSerializer(OsVersionSerializer osVersionSerializer, NodeVersionSerializer nodeVersionSerializer) { this.osVersionSerializer = Objects.requireNonNull(osVersionSerializer, "osVersionSerializer must be non-null"); + this.nodeVersionSerializer = Objects.requireNonNull(nodeVersionSerializer, "nodeVersionSerializer must be non-null"); } public Slime toSlime(OsVersionStatus status) { @@ -53,6 +52,8 @@ public class OsVersionStatusSerializer { status.versions().forEach((version, nodes) -> { Cursor object = versions.addObject(); osVersionSerializer.toSlime(version, object); + nodeVersionSerializer.nodeVersionsToSlime(nodes, object.setArray(nodeVersionsField)); + // TODO(mpolden): Stop writing this after September 2019 nodesToSlime(nodes, object.setArray(nodesField)); }); return slime; @@ -62,40 +63,33 @@ public class OsVersionStatusSerializer { return new OsVersionStatus(osVersionsFromSlime(slime.get().field(versionsField))); } - private void nodesToSlime(List nodes, Cursor array) { - nodes.forEach(node -> nodeToSlime(node, array.addObject())); + private void nodesToSlime(NodeVersions nodeVersions, Cursor array) { + nodeVersions.asMap().values().forEach(node -> nodeToSlime(node, array.addObject())); } - private void nodeToSlime(OsVersionStatus.Node node, Cursor object) { + private void nodeToSlime(NodeVersion node, Cursor object) { object.setString(hostnameField, node.hostname().value()); - object.setString(versionField, node.version().toFullString()); - object.setString(regionField, node.region().value()); - object.setString(environmentField, node.environment().value()); + object.setString(versionField, node.currentVersion().toFullString()); + object.setString(regionField, node.zone().region().value()); + object.setString(environmentField, node.zone().environment().value()); } - private Map> osVersionsFromSlime(Inspector array) { - Map> versions = new TreeMap<>(); + private ImmutableMap osVersionsFromSlime(Inspector array) { + var versions = ImmutableSortedMap.naturalOrder(); array.traverse((ArrayTraverser) (i, object) -> { OsVersion osVersion = osVersionSerializer.fromSlime(object); - List nodes = nodesFromSlime(object.field(nodesField)); - versions.put(osVersion, nodes); + versions.put(osVersion, nodesFromSlime(object, osVersion.version())); }); - return Collections.unmodifiableMap(versions); + return versions.build(); } - private List nodesFromSlime(Inspector array) { - List nodes = new ArrayList<>(); - array.traverse((ArrayTraverser) (i, object) -> nodes.add(nodeFromSlime(object))); - return Collections.unmodifiableList(nodes); - } - - private OsVersionStatus.Node nodeFromSlime(Inspector object) { - return new OsVersionStatus.Node( - HostName.from(object.field(hostnameField).asString()), - Version.fromString(object.field(versionField).asString()), - Environment.from(object.field(environmentField).asString()), - RegionName.from(object.field(regionField).asString()) - ); + // TODO(mpolden): Simplify and in-line after September 2019 + private NodeVersions nodesFromSlime(Inspector object, Version version) { + var newField = object.field(nodeVersionsField); + if (newField.valid()) { + return nodeVersionSerializer.nodeVersionsFromSlime(newField, version); + } + return nodeVersionSerializer.nodeVersionsFromSlime(object.field(nodesField), version); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index fd02546cf58..4373c8977de 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -10,7 +9,6 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; -import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -22,7 +20,6 @@ import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Set; /** @@ -95,7 +92,7 @@ public class VersionStatusSerializer { } private void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) { - nodeVersionSerializer.nodeVersionsToSlime(nodeVersions.asMap().values(), array, false); + nodeVersionSerializer.nodeVersionsToSlime(nodeVersions, array); } private void deploymentStatisticsToSlime(DeploymentStatistics statistics, Cursor object) { @@ -123,19 +120,11 @@ public class VersionStatusSerializer { object.field(isControllerVersionField).asBool(), object.field(isSystemVersionField).asBool(), object.field(isReleasedField).asBool(), - nodeVersionsFromSlime(object.field(nodeVersionsField), deploymentStatistics.version()), + nodeVersionSerializer.nodeVersionsFromSlime(object.field(nodeVersionsField), deploymentStatistics.version()), VespaVersion.Confidence.valueOf(object.field(confidenceField).asString()) ); } - private NodeVersions nodeVersionsFromSlime(Inspector object, Version version) { - var nodeVersions = ImmutableMap.builder(); - for (var nodeVersion : nodeVersionSerializer.nodeVersionsFromSlime(object, Optional.of(version))) { - nodeVersions.put(nodeVersion.hostname(), nodeVersion); - } - return new NodeVersions(nodeVersions.build()); - } - private Set configServersFromSlime(Inspector array) { Set configServerHostnames = new LinkedHashSet<>(); array.traverse((ArrayTraverser) (i, entry) -> configServerHostnames.add(HostName.from(entry.asString()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index 450f4481c5f..c168a057bfb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -160,17 +160,17 @@ public class OsApiHandler extends AuditLoggingRequestHandler { Set osVersions = controller.osVersions(); Cursor versions = root.setArray("versions"); - controller.osVersionStatus().versions().forEach((osVersion, nodes) -> { + controller.osVersionStatus().versions().forEach((osVersion, nodeVersions) -> { Cursor currentVersionObject = versions.addObject(); currentVersionObject.setString("version", osVersion.version().toFullString()); currentVersionObject.setBool("targetVersion", osVersions.contains(osVersion)); currentVersionObject.setString("cloud", osVersion.cloud().value()); Cursor nodesArray = currentVersionObject.setArray("nodes"); - nodes.forEach(node -> { + nodeVersions.asMap().values().forEach(nodeVersion -> { Cursor nodeObject = nodesArray.addObject(); - nodeObject.setString("hostname", node.hostname().value()); - nodeObject.setString("environment", node.environment().value()); - nodeObject.setString("region", node.region().value()); + nodeObject.setString("hostname", nodeVersion.hostname().value()); + nodeObject.setString("environment", nodeVersion.zone().environment().value()); + nodeObject.setString("region", nodeVersion.zone().region().value()); }); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index a73a20198f0..d5e83d99cdd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -4,9 +4,7 @@ package com.yahoo.vespa.hosted.controller.versions; import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -14,11 +12,11 @@ import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -29,25 +27,25 @@ import java.util.stream.Collectors; */ public class OsVersionStatus { - public static final OsVersionStatus empty = new OsVersionStatus(Collections.emptyMap()); + public static final OsVersionStatus empty = new OsVersionStatus(ImmutableMap.of()); - private final Map> versions; + private final Map versions; /** Public for serialization purpose only. Use {@link OsVersionStatus#compute(Controller)} for an up-to-date status */ - public OsVersionStatus(Map> versions) { + public OsVersionStatus(ImmutableMap versions) { this.versions = ImmutableMap.copyOf(Objects.requireNonNull(versions, "versions must be non-null")); } /** All known OS versions and their nodes */ - public Map> versions() { + public Map versions() { return versions; } /** Returns nodes eligible for OS upgrades that exist in given cloud */ - public List nodesIn(CloudName cloud) { + public List nodesIn(CloudName cloud) { return versions.entrySet().stream() .filter(entry -> entry.getKey().cloud().equals(cloud)) - .flatMap(entry -> entry.getValue().stream()) + .flatMap(entry -> entry.getValue().asMap().values().stream()) .collect(Collectors.toUnmodifiableList()); } @@ -61,28 +59,52 @@ public class OsVersionStatus { /** Compute the current OS versions in this system. This is expensive and should be called infrequently */ public static OsVersionStatus compute(Controller controller) { - Map> versions = new HashMap<>(); - - // Always include all target versions - controller.osVersions().forEach(osVersion -> versions.put(osVersion, new ArrayList<>())); - - for (SystemApplication application : SystemApplication.all()) { - if (!application.isEligibleForOsUpgrades()) { - continue; // Avoid querying applications that are not eligible for OS upgrades - } - for (ZoneApi zone : zonesToUpgrade(controller)) { - controller.serviceRegistry().configServer().nodeRepository().list(zone.getId(), application.id()).stream() + var osVersionStatus = controller.osVersionStatus(); + var osVersions = new HashMap>(); + var now = controller.clock().instant(); + controller.osVersions().forEach(osVersion -> osVersions.put(osVersion, new ArrayList<>())); + + for (var application : SystemApplication.all()) { + if (!application.isEligibleForOsUpgrades()) continue; + for (var zone : zonesToUpgrade(controller)) { + var targetOsVersion = controller.serviceRegistry().configServer().nodeRepository() + .targetVersionsOf(zone.getId()) + .osVersion(application.nodeType()) + .orElse(Version.emptyVersion); + controller.serviceRegistry().configServer().nodeRepository() + .list(zone.getId(), application.id()).stream() .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) - .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.getEnvironment(), zone.getRegionName())) - .forEach(node -> { - var version = new OsVersion(node.version(), zone.getCloudName()); - versions.putIfAbsent(version, new ArrayList<>()); - versions.get(version).add(node); + .map(node -> new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), targetOsVersion, now)) + .forEach(nodeVersion -> { + var newNodeVersion = osVersionStatus.of(nodeVersion.hostname()) + .map(nv -> nv.withCurrentVersion(nodeVersion.currentVersion(), now) + .withWantedVersion(nodeVersion.wantedVersion())) + .orElse(nodeVersion); + var version = new OsVersion(newNodeVersion.currentVersion(), zone.getCloudName()); + osVersions.putIfAbsent(version, new ArrayList<>()); + osVersions.get(version).add(newNodeVersion); }); } } - return new OsVersionStatus(versions); + var newOsVersions = ImmutableMap.builder(); + for (var osVersion : osVersions.entrySet()) { + var nodeVersions = ImmutableMap.builder(); + for (var nodeVersion : osVersion.getValue()) { + nodeVersions.put(nodeVersion.hostname(), nodeVersion); + } + newOsVersions.put(osVersion.getKey(), new NodeVersions(nodeVersions.build())); + } + return new OsVersionStatus(newOsVersions.build()); + } + + /** Returns version of node identified by given host name */ + private Optional of(HostName hostname) { + return versions.values().stream() + .map(nodeVersions -> nodeVersions.asMap().get(hostname)) + .map(Optional::ofNullable) + .flatMap(Optional::stream) + .findFirst(); } private static List zonesToUpgrade(Controller controller) { @@ -92,52 +114,4 @@ public class OsVersionStatus { .collect(Collectors.toUnmodifiableList()); } - /** A node in this system and its current OS version */ - public static class Node { - - private final HostName hostname; - private final Version version; - private final Environment environment; - private final RegionName region; - - public Node(HostName hostname, Version version, Environment environment, RegionName region) { - this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); - this.version = Objects.requireNonNull(version, "version must be non-null"); - this.environment = Objects.requireNonNull(environment, "environment must be non-null"); - this.region = Objects.requireNonNull(region, "region must be non-null"); - } - - public HostName hostname() { - return hostname; - } - - public Version version() { - return version; - } - - public Environment environment() { - return environment; - } - - public RegionName region() { - return region; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Node node = (Node) o; - return Objects.equals(hostname, node.hostname) && - Objects.equals(version, node.version) && - environment == node.environment && - Objects.equals(region, node.region); - } - - @Override - public int hashCode() { - return Objects.hash(hostname, version, environment, region); - } - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 1af5fafbb79..5e92112d465 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import org.junit.Before; import org.junit.Test; @@ -111,13 +111,13 @@ public class OsUpgraderTest { assertWanted(version1, SystemApplication.tenantHost, zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()); statusUpdater.maintain(); assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud).stream() - .allMatch(node -> node.version().equals(version1))); + .allMatch(node -> node.currentVersion().equals(version1))); } - private List nodesOn(Version version) { + private List nodesOn(Version version) { return tester.controller().osVersionStatus().versions().entrySet().stream() .filter(entry -> entry.getKey().version().equals(version)) - .flatMap(entry -> entry.getValue().stream()) + .flatMap(entry -> entry.getValue().asMap().values().stream()) .collect(Collectors.toList()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java index fe7f39fd66d..e51fcff33d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -3,18 +3,15 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.config.provision.zone.UpgradePolicy; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import org.junit.Test; import java.time.Duration; -import java.util.List; -import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -47,10 +44,10 @@ public class OsVersionStatusUpdaterTest { tester.controller().upgradeOsIn(cloud, version1, false); statusUpdater.maintain(); - Map> osVersions = tester.controller().osVersionStatus().versions(); + var osVersions = tester.controller().osVersionStatus().versions(); assertEquals(2, osVersions.size()); - assertFalse("All nodes on unknown version", osVersions.get(new OsVersion(Version.emptyVersion, cloud)).isEmpty()); - assertTrue("No nodes on current target", osVersions.get(new OsVersion(version1, cloud)).isEmpty()); + assertFalse("All nodes on unknown version", osVersions.get(new OsVersion(Version.emptyVersion, cloud)).asMap().isEmpty()); + assertTrue("No nodes on current target", osVersions.get(new OsVersion(version1, cloud)).asMap().isEmpty()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java index 5073f651fd3..c60137e47b4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java @@ -1,18 +1,23 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import org.junit.Test; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Instant; import java.util.List; -import java.util.Map; -import java.util.TreeMap; import static org.junit.Assert.assertEquals; @@ -25,22 +30,41 @@ public class OsVersionStatusSerializerTest { public void test_serialization() { Version version1 = Version.fromString("7.1"); Version version2 = Version.fromString("7.2"); - Map> versions = new TreeMap<>(); + var versions = ImmutableMap.builder(); - versions.put(new OsVersion(version1, CloudName.defaultName()), List.of( - new OsVersionStatus.Node(HostName.from("node1"), version1, Environment.prod, RegionName.from("us-west")), - new OsVersionStatus.Node(HostName.from("node2"), version1, Environment.prod, RegionName.from("us-east")) - )); - versions.put(new OsVersion(version2, CloudName.defaultName()), List.of( - new OsVersionStatus.Node(HostName.from("node3"), version2, Environment.prod, RegionName.from("us-west")), - new OsVersionStatus.Node(HostName.from("node4"), version2, Environment.prod, RegionName.from("us-east")) + versions.put(new OsVersion(version1, CloudName.defaultName()), NodeVersions.EMPTY.with(List.of( + new NodeVersion(HostName.from("node1"), ZoneId.from("prod", "us-west"), version1, version2, Instant.ofEpochMilli(1)), + new NodeVersion(HostName.from("node2"), ZoneId.from("prod", "us-east"), version1, version2, Instant.ofEpochMilli(2)) + ))); + versions.put(new OsVersion(version2, CloudName.defaultName()), NodeVersions.EMPTY.with(List.of( + new NodeVersion(HostName.from("node3"), ZoneId.from("prod", "us-west"), version2, version2, Instant.ofEpochMilli(3)), + new NodeVersion(HostName.from("node4"), ZoneId.from("prod", "us-east"), version2, version2, Instant.ofEpochMilli(4)) + ))); - )); - - OsVersionStatusSerializer serializer = new OsVersionStatusSerializer(new OsVersionSerializer()); - OsVersionStatus status = new OsVersionStatus(versions); + OsVersionStatusSerializer serializer = new OsVersionStatusSerializer(new OsVersionSerializer(), new NodeVersionSerializer()); + OsVersionStatus status = new OsVersionStatus(versions.build()); OsVersionStatus serialized = serializer.fromSlime(serializer.toSlime(status)); assertEquals(status.versions(), serialized.versions()); } + @Test + public void testLegacySerialization() throws Exception { + var data = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/os-version-status-legacy-format.json")); + var serializer = new OsVersionStatusSerializer(new OsVersionSerializer(), new NodeVersionSerializer()); + var versions = ImmutableMap.of( + new OsVersion(Version.fromString("7.42"), CloudName.from("yahoo")), + NodeVersions.EMPTY.with(List.of(new NodeVersion(HostName.from("node1"), ZoneId.from("prod", "us-north-1"), + Version.fromString("7.42"), Version.emptyVersion, Instant.EPOCH), + new NodeVersion(HostName.from("node2"), ZoneId.from("prod", "us-north-2"), + Version.fromString("7.42"), Version.emptyVersion, Instant.EPOCH)))); + + var deserialized = serializer.fromSlime(SlimeUtils.jsonToSlime(data)); + assertEquals(versions, deserialized.versions()); + + + var serialized = new String(SlimeUtils.toJsonBytes(serializer.toSlime(new OsVersionStatus(versions))), StandardCharsets.UTF_8); + assertEquals("{\"versions\":[{\"version\":\"7.42.0\",\"cloud\":\"yahoo\",\"nodeVersions\":[{\"hostname\":\"node1\",\"zone\":\"prod.us-north-1\",\"wantedVersion\":\"0.0.0\",\"changedAt\":0},{\"hostname\":\"node2\",\"zone\":\"prod.us-north-2\",\"wantedVersion\":\"0.0.0\",\"changedAt\":0}],\"nodes\":[{\"hostname\":\"node1\",\"version\":\"7.42.0\",\"region\":\"us-north-1\",\"environment\":\"prod\"},{\"hostname\":\"node2\",\"version\":\"7.42.0\",\"region\":\"us-north-2\",\"environment\":\"prod\"}]}]}", + serialized); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/os-version-status-legacy-format.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/os-version-status-legacy-format.json new file mode 100644 index 00000000000..5a6a864cbf8 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/os-version-status-legacy-format.json @@ -0,0 +1,22 @@ +{ + "versions": [ + { + "version": "7.42", + "cloud": "yahoo", + "nodes": [ + { + "hostname": "node1", + "version": "7.42", + "region": "us-north-1", + "environment": "prod" + }, + { + "hostname": "node2", + "version": "7.42", + "region": "us-north-2", + "environment": "test" + } + ] + } + ] +} -- cgit v1.2.3