diff options
14 files changed, 328 insertions, 17 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 44c6d899360..86ae71f747c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -43,11 +43,14 @@ public interface ConfigServer { JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds); ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region); - + Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath); - - /** Returns the version this particular config server is running */ - Version version(URI configServerUri); + + /** Returns the version of this config server */ + ConfigServerVersion version(URI configServerUri); + + /** Upgrade config server at URI to the given version */ + void upgrade(URI configServerUri, Version version); /** * Set new status on en endpoint in one zone. diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerVersion.java new file mode 100644 index 00000000000..ca219d4e802 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerVersion.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import com.yahoo.component.Version; + +/** + * Represents the config server's current and wanted version. + * + * @author mpolden + */ +public class ConfigServerVersion { + + private final Version current; + private final Version wanted; + + public ConfigServerVersion(Version current, Version wanted) { + this.current = current; + this.wanted = wanted; + } + + public Version current() { + return current; + } + + public Version wanted() { + return wanted; + } + + public boolean upgrading() { + return !current.equals(wanted); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/UpgradePolicy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/UpgradePolicy.java new file mode 100644 index 00000000000..96ff468d2d6 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/UpgradePolicy.java @@ -0,0 +1,46 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.zone; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * This class declares the order to use when upgrading zones in a system. + * + * @author mpolden + */ +public class UpgradePolicy { + + private final List<List<ZoneId>> zones; + + private UpgradePolicy(List<List<ZoneId>> zones) { + this.zones = zones; + } + + public List<List<ZoneId>> asList() { + return Collections.unmodifiableList(zones); + } + + private UpgradePolicy with(ZoneId... zone) { + List<List<ZoneId>> zones = new ArrayList<>(this.zones); + zones.add(Arrays.asList(zone)); + return new UpgradePolicy(zones); + } + + /** Upgrade given zone as the next step */ + public UpgradePolicy upgrade(ZoneId zone) { + return with(zone); + } + + /** Upgrade given zones in parallel as the next step */ + public UpgradePolicy upgradeInParallel(ZoneId... zone) { + return with(zone); + } + + public static UpgradePolicy create() { + return new UpgradePolicy(Collections.emptyList()); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index 169f38aaf4c..dea14493d26 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.zone; import com.yahoo.config.provision.Environment; @@ -28,7 +28,7 @@ public interface ZoneRegistry { /** Returns the default region for the given environment, if one is configured */ Optional<RegionName> getDefaultRegion(Environment environment); - /** Returns a list with all known config servers in the given zone, with a secure connection URL */ + /** Returns the API endpoints of all known config servers in the given zone */ List<URI> getConfigServerUris(ZoneId zoneId); /** Returns the URI for the config server VIP in the given zone, or Optional.empty() if no VIP exists */ @@ -49,4 +49,7 @@ public interface ZoneRegistry { /** Return the configserver's Athenz service identity */ AthenzService getConfigServerAthenzService(ZoneId zoneId); + /** Returns the upgrade policy to use for zones in this registry */ + UpgradePolicy upgradePolicy(); + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index b08e5167e67..0746b479f27 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.AbstractComponent; @@ -36,6 +36,7 @@ public class ControllerMaintenance extends AbstractComponent { private final DeploymentMetricsMaintainer deploymentMetricsMaintainer; private final ApplicationOwnershipConfirmer applicationOwnershipConfirmer; private final DnsMaintainer dnsMaintainer; + private final SystemUpgrader systemUpgrader; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator, @@ -56,6 +57,7 @@ public class ControllerMaintenance extends AbstractComponent { deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); + systemUpgrader = new SystemUpgrader(controller, maintenanceInterval, jobControl); } public Upgrader upgrader() { return upgrader; } @@ -77,6 +79,7 @@ public class ControllerMaintenance extends AbstractComponent { deploymentMetricsMaintainer.deconstruct(); applicationOwnershipConfirmer.deconstruct(); dnsMaintainer.deconstruct(); + systemUpgrader.maintain(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java new file mode 100644 index 00000000000..3a729bb9657 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -0,0 +1,76 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; + +import java.net.URI; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Maintenance job which schedules upgrades of the system. + * + * @author mpolden + */ +public class SystemUpgrader extends Maintainer { + + private static final Logger log = Logger.getLogger(SystemUpgrader.class.getName()); + + public SystemUpgrader(Controller controller, Duration interval, JobControl jobControl) { + super(controller, interval, jobControl); + } + + @Override + protected void maintain() { + Optional<Version> target = targetVersion(); + if (!target.isPresent()) { + return; + } + for (List<ZoneId> zones : controller().zoneRegistry().upgradePolicy().asList()) { + if (!completeUpgrade(zones, target.get())) { + break; + } + } + } + + /** Returns true if upgrade of given zones is complete */ + private boolean completeUpgrade(List<ZoneId> zones, Version version) { + boolean completed = true; + for (ZoneId zone : zones) { + startUpgrade(zone, version); + completed = completed && !isUpgrading(zone); + } + return completed; + } + + /** Returns true if any config servers in given zone are upgrading */ + private boolean isUpgrading(ZoneId zone) { + return configServerUris(zone).stream().anyMatch(uri -> controller().configServer().version(uri).upgrading()); + } + + /** Schedule upgrade of config servers in given zone, if necessary */ + private void startUpgrade(ZoneId zone, Version version) { + configServerUris(zone).stream() + .filter(uri -> !controller().configServer().version(uri).wanted().equals(version)) + .peek(uri -> log.info(String.format("Upgrading config server %s in %s", uri.getHost(), + zone))) + .forEach(uri -> controller().configServer().upgrade(uri, version)); + } + + /** Returns target version for the system */ + private Optional<Version> targetVersion() { + return controller().versionStatus().controllerVersion() + .filter(vespaVersion -> !vespaVersion.isSystemVersion()) + .map(VespaVersion::versionNumber); + } + + private List<URI> configServerUris(ZoneId zone) { + return controller().zoneRegistry().getConfigServerUris(zone); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 05881d2c466..4ebe5a2de7a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -133,7 +133,7 @@ public class VersionStatus { ListMap<Version, HostName> versions = new ListMap<>(); for (URI configServer : configServers) - versions.put(controller.configServer().version(configServer), + versions.put(controller.configServer().version(configServer).current(), HostName.from(configServer.getHost())); return versions; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java index 576bb33687b..1f96fc3ef7f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerVersion; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.serviceview.bindings.ApplicationView; @@ -38,6 +39,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map<ApplicationId, Boolean> applicationActivated = new HashMap<>(); private final Map<String, EndpointStatus> endpoints = new HashMap<>(); private final Map<URI, Version> versions = new HashMap<>(); + private final Map<URI, Version> wantedVersions = new HashMap<>(); private Version defaultVersion = new Version(6, 1, 0); private Version lastPrepareVersion = null; @@ -66,6 +68,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return versions; } + @Override + public void upgrade(URI configServerUri, Version version) { + wantedVersions.put(configServerUri, version); + } + /** Set the default config server version */ public void setDefaultVersion(Version version) { this.defaultVersion = version; @@ -171,8 +178,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public Version version(URI configServerUri) { - return versions.getOrDefault(configServerUri, defaultVersion); + public ConfigServerVersion version(URI configServerUri) { + return new ConfigServerVersion(versions.getOrDefault(configServerUri, defaultVersion), + wantedVersions.getOrDefault(configServerUri, defaultVersion)); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index ee426557596..81228749323 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -561,7 +561,7 @@ public class ControllerTest { @Test public void testCleanupOfStaleDeploymentData() throws IOException { DeploymentTester tester = new DeploymentTester(); - tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); + tester.controllerTester().zoneRegistry().setSystemName(SystemName.cd); tester.controllerTester().zoneRegistry().setZones(ZoneId.from("prod", "cd-us-central-1")); Supplier<Map<JobType, JobStatus>> statuses = () -> @@ -769,7 +769,7 @@ public class ControllerTest { @Test public void testDeployWithoutProjectId() { DeploymentTester tester = new DeploymentTester(); - tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); + tester.controllerTester().zoneRegistry().setSystemName(SystemName.cd); tester.controllerTester().zoneRegistry().setZones(ZoneId.from("prod", "cd-us-central-1")); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java index 53e2d3ca895..28388a48c37 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.inject.Inject; @@ -6,6 +6,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.athenz.api.AthenzService; @@ -32,6 +33,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); private List<ZoneId> zones = new ArrayList<>(); private SystemName system = SystemName.main; + private UpgradePolicy upgradePolicy = null; @Inject public ZoneRegistryMock() { @@ -61,11 +63,16 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return setZones(Arrays.asList(zone)); } - public ZoneRegistryMock setSystem(SystemName system) { + public ZoneRegistryMock setSystemName(SystemName system) { this.system = system; return this; } + public ZoneRegistryMock setUpgradePolicy(UpgradePolicy upgradePolicy) { + this.upgradePolicy = upgradePolicy; + return this; + } + @Override public SystemName system() { return system; @@ -81,6 +88,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override + public UpgradePolicy upgradePolicy() { + return upgradePolicy; + } + + @Override public boolean hasZone(ZoneId zoneId) { return zones.contains(zoneId); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index d0e366663a1..d5154c95cdc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; +import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; @@ -46,6 +47,7 @@ public class DeploymentTester { private final ControllerTester tester; private final Upgrader upgrader; + private final SystemUpgrader systemUpgrader; private final ReadyJobsTrigger readyJobTrigger; public DeploymentTester() { @@ -57,10 +59,15 @@ public class DeploymentTester { tester.curator().writeUpgradesPerMinute(100); this.upgrader = new Upgrader(tester.controller(), maintenanceInterval, new JobControl(tester.curator()), tester.curator()); + this.systemUpgrader = new SystemUpgrader(tester.controller(), maintenanceInterval, new JobControl(tester.curator())); this.readyJobTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval, new JobControl(tester.curator())); } + public SystemUpgrader systemUpgrader() { + return systemUpgrader; + } + public Upgrader upgrader() { return upgrader; } public ReadyJobsTrigger readyJobTrigger() { return readyJobTrigger; } @@ -94,11 +101,16 @@ public class DeploymentTester { } public void updateVersionStatus(Version newVersion) { - controller().curator().writeControllerVersion(controller().hostname(), newVersion); + upgradeController(newVersion); configServer().setDefaultVersion(newVersion); controller().updateVersionStatus(VersionStatus.compute(controller())); } + public void upgradeController(Version newVersion) { + controller().curator().writeControllerVersion(controller().hostname(), newVersion); + controller().updateVersionStatus(VersionStatus.compute(controller())); + } + public void upgradeSystem(Version version) { updateVersionStatus(version); upgrader().maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index 0a330bfa4c5..d28a70eb838 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; @@ -137,7 +137,7 @@ public class FailureRedeployerTest { @Test public void ignoresPullRequestInstances() throws Exception { DeploymentTester tester = new DeploymentTester(); - tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); + tester.controllerTester().zoneRegistry().setSystemName(SystemName.cd); // Current system version, matches version in test data Version version = Version.fromString("6.42.1"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java new file mode 100644 index 00000000000..8ad4b1a4fd9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -0,0 +1,113 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import org.junit.Test; + +import java.net.URI; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author mpolden + */ +public class SystemUpgraderTest { + + private final DeploymentTester tester = new DeploymentTester(); + + @Test + public void upgrade_system() { + ZoneId zone1 = ZoneId.from("prod", "eu-west-1"); + ZoneId zone2 = ZoneId.from("prod", "us-west-1"); + ZoneId zone3 = ZoneId.from("prod", "us-central-1"); + ZoneId zone4 = ZoneId.from("prod", "us-east-3"); + + tester.controllerTester().zoneRegistry().setUpgradePolicy( + UpgradePolicy.create() + .upgrade(zone1) + .upgradeInParallel(zone2, zone3) + .upgrade(zone4) + ); + + Version version1 = Version.fromString("6.5"); + tester.updateVersionStatus(version1); + tester.systemUpgrader().maintain(); + assertTrue("Zones are on current version", onVersion(version1, zone1, zone2, zone3, zone4)); + + // Controller upgrades + Version version2 = Version.fromString("6.6"); + tester.upgradeController(version2); + assertEquals(version2, tester.controller().versionStatus().controllerVersion().get().versionNumber()); + + // System upgrade scheduled. First zone starts upgrading + tester.systemUpgrader().maintain(); + assertWantedVersion(version2, zone1); + assertWantedVersion(version1, zone2, zone3, zone4); // Other zones remains on previous version + + // Zones 1 completes upgrade + completeUpgrade(version2, zone1); + + // Zones 2 and 3 upgrade in parallel + tester.systemUpgrader().maintain(); + assertWantedVersion(version2, zone2, zone3); + assertWantedVersion(version1, zone4); + completeUpgrade(version2, zone2, zone3); + + // Zone 4 upgrades last + tester.systemUpgrader().maintain(); + assertWantedVersion(version2, zone4); + completeUpgrade(version2, zone4); + + // Next run does nothing as system is now upgraded + tester.systemUpgrader().maintain(); + assertWantedVersion(version2, zone1, zone2, zone3, zone4); + assertTrue(onVersion(version2, zone1, zone2, zone3, zone4)); + } + + @Test + public void never_downgrades_system() { + ZoneId zone = ZoneId.from("prod", "eu-west-1"); + tester.controllerTester().zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(zone)); + + Version version = Version.fromString("6.5"); + tester.updateVersionStatus(version); + tester.systemUpgrader().maintain(); + assertTrue("Zone is on current version", onVersion(version, zone)); + + // Controller is downgraded + tester.upgradeController(Version.fromString("6.4")); + + // Wanted version for zone remains unchanged + tester.systemUpgrader().maintain(); + assertWantedVersion(version, zone); + } + + private void completeUpgrade(Version version, ZoneId... zones) { + for (ZoneId zone : zones) { + for (URI configServer : tester.controller().zoneRegistry().getConfigServerUris(zone)) { + tester.controllerTester().configServer().versions().put(configServer, version); + } + assertTrue(onVersion(version, zone)); + } + } + + private void assertWantedVersion(Version version, ZoneId... zones) { + for (ZoneId zone : zones) { + for (URI configServer : tester.controller().zoneRegistry().getConfigServerUris(zone)) { + assertEquals(version, tester.controller().configServer().version(configServer).wanted()); + } + } + } + + private boolean onVersion(Version version, ZoneId... zone) { + return Arrays.stream(zone) + .flatMap(z -> tester.controller().zoneRegistry().getConfigServerUris(z).stream()) + .allMatch(uri -> tester.controller().configServer().version(uri).current().equals(version)); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 8f8b76c83c6..0ad82df8db1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -31,6 +31,9 @@ "name": "ReadyJobsTrigger" }, { + "name": "SystemUpgrader" + }, + { "name": "Upgrader" }, { |