From 5099a110e002ee2508a1082fa4e8f37fa264dbce Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 30 May 2018 22:17:20 +0200 Subject: Upgrade hosts --- .../hosted/controller/maintenance/SystemUpgrader.java | 18 ++++++++++-------- .../controller/maintenance/SystemUpgraderTest.java | 1 - 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 516cd52d710..35adb632c38 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -38,9 +37,9 @@ public class SystemUpgrader extends Maintainer { if (!target.isPresent()) { return; } - // TODO: Change to SystemApplication.all() once host applications support upgrade + try { - deploy(Arrays.asList(SystemApplication.configServer, SystemApplication.zone), target.get()); + deploy(SystemApplication.all(), target.get()); } catch (Exception e) { log.log(Level.WARNING, "Failed to upgrade system. Retrying in " + maintenanceInterval(), e); } @@ -53,12 +52,11 @@ public class SystemUpgrader extends Maintainer { for (ZoneId zone : zones) { for (SystemApplication application : applications) { boolean dependenciesConverged = application.dependencies().stream() - .filter(applications::contains) // TODO: Remove when all() is used. - .allMatch(dependency -> currentVersion(zone, dependency.id()).equals(target)); + .allMatch(dependency -> convergedOn(zone, dependency, target)); if (dependenciesConverged) { deploy(target, application, zone); } - converged &= currentVersion(zone, application.id()).equals(target); + converged &= convergedOn(zone, application, target); } } if (!converged) { @@ -75,12 +73,16 @@ public class SystemUpgrader extends Maintainer { } } + private boolean convergedOn(ZoneId zone, SystemApplication application, Version target) { + return currentVersion(zone, application.id(), target).equals(target); + } + private Version wantedVersion(ZoneId zone, ApplicationId application, Version defaultVersion) { return minVersion(zone, application, Node::wantedVersion).orElse(defaultVersion); } - private Version currentVersion(ZoneId zone, ApplicationId application) { - return minVersion(zone, application, Node::currentVersion).orElse(Version.emptyVersion); + private Version currentVersion(ZoneId zone, ApplicationId application, Version defaultVersion) { + return minVersion(zone, application, Node::currentVersion).orElse(defaultVersion); } private Optional minVersion(ZoneId zone, ApplicationId application, Function versionField) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 1ec64f8d478..4b563ed203d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -121,7 +121,6 @@ public class SystemUpgraderTest { } @Test - @Ignore // TODO: Unignore once host applications support upgrade public void upgrade_system_containing_host_applications() { tester.controllerTester().zoneRegistry().setUpgradePolicy( UpgradePolicy.create() -- cgit v1.2.3 From b7c8f12fb1a6d27505f5400e4077c9f46d27c173 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 30 May 2018 23:22:31 +0200 Subject: Handle unreachable node-repository when getting versions --- .../controller/maintenance/SystemUpgrader.java | 45 ++++++++++++++-------- 1 file changed, 30 insertions(+), 15 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 35adb632c38..ddea4e1e387 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -38,11 +38,7 @@ public class SystemUpgrader extends Maintainer { return; } - try { - deploy(SystemApplication.all(), target.get()); - } catch (Exception e) { - log.log(Level.WARNING, "Failed to upgrade system. Retrying in " + maintenanceInterval(), e); - } + deploy(SystemApplication.all(), target.get()); } /** Deploy a list of system applications until they converge on the given version */ @@ -50,13 +46,14 @@ public class SystemUpgrader extends Maintainer { for (List zones : controller().zoneRegistry().upgradePolicy().asList()) { boolean converged = true; for (ZoneId zone : zones) { - for (SystemApplication application : applications) { - boolean dependenciesConverged = application.dependencies().stream() - .allMatch(dependency -> convergedOn(zone, dependency, target)); - if (dependenciesConverged) { - deploy(target, application, zone); - } - converged &= convergedOn(zone, application, target); + try { + converged &= deployInZone(zone, applications, target); + } catch (UnreachableNodeRepositoryException e) { + converged = false; + log.log(Level.WARNING, e.getMessage() + ". Continuing to next parallel deployed zone"); + } catch (Exception e) { + converged = false; + log.log(Level.WARNING, "Failed to upgrade " + zone + ". Continuing to next parallel deployed zone", e); } } if (!converged) { @@ -65,6 +62,20 @@ public class SystemUpgrader extends Maintainer { } } + /** @return true if all applications have converged to the target version in the zone */ + private boolean deployInZone(ZoneId zone, List applications, Version target) { + boolean converged = true; + for (SystemApplication application : applications) { + boolean dependenciesConverged = application.dependencies().stream() + .allMatch(dependency -> convergedOn(zone, dependency, target)); + if (dependenciesConverged) { + deploy(target, application, zone); + } + converged &= convergedOn(zone, application, target); + } + return converged; + } + /** Deploy application on given version idempotently */ private void deploy(Version target, SystemApplication application, ZoneId zone) { if (!wantedVersion(zone, application.id(), target).equals(target)) { @@ -94,9 +105,8 @@ public class SystemUpgrader extends Maintainer { .map(versionField) .min(Comparator.naturalOrder()); } catch (Exception e) { - log.log(Level.WARNING, String.format("Failed to get version for %s in %s: %s", application, zone, - Exceptions.toMessageString(e))); - return Optional.empty(); + throw new UnreachableNodeRepositoryException(String.format("Failed to get version for %s in %s: %s", + application, zone, Exceptions.toMessageString(e))); } } @@ -107,4 +117,9 @@ public class SystemUpgrader extends Maintainer { .map(VespaVersion::versionNumber); } + private class UnreachableNodeRepositoryException extends RuntimeException { + private UnreachableNodeRepositoryException(String reason) { + super(reason); + } + } } -- cgit v1.2.3 From d7bb6cb9d29dec33f7a93a06a41c0a36caf26871 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 31 May 2018 09:32:47 +0200 Subject: Re-order argumnets, add convergedOn() for List of Applications --- .../vespa/hosted/controller/maintenance/SystemUpgrader.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index ddea4e1e387..ac0d08f5105 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -66,12 +66,10 @@ public class SystemUpgrader extends Maintainer { private boolean deployInZone(ZoneId zone, List applications, Version target) { boolean converged = true; for (SystemApplication application : applications) { - boolean dependenciesConverged = application.dependencies().stream() - .allMatch(dependency -> convergedOn(zone, dependency, target)); - if (dependenciesConverged) { + if (convergedOn(target, application.dependencies(), zone)) { deploy(target, application, zone); } - converged &= convergedOn(zone, application, target); + converged &= convergedOn(target, application, zone); } return converged; } @@ -84,7 +82,11 @@ public class SystemUpgrader extends Maintainer { } } - private boolean convergedOn(ZoneId zone, SystemApplication application, Version target) { + private boolean convergedOn(Version target, List applications, ZoneId zone) { + return applications.stream().allMatch(application -> convergedOn(target, application, zone)); + } + + private boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { return currentVersion(zone, application.id(), target).equals(target); } -- cgit v1.2.3