diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-03-24 14:40:27 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-03-24 14:40:27 +0100 |
commit | 8f28065244baefad08b8d9894c67c29f16b14354 (patch) | |
tree | 18d94f3a70b62f537ed94787657f7bb42d380a0c /node-repository | |
parent | c84c3a36027e7e73ca87f0217f1c311430710f3a (diff) |
Spread deployments evently over the maintenance period
Diffstat (limited to 'node-repository')
2 files changed, 14 insertions, 7 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 72c90aed458..28d6297ca18 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -9,6 +9,8 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.Executor; @@ -39,7 +41,8 @@ public class ApplicationMaintainer extends Maintainer { @Override protected void maintain() { - for (ApplicationId application : activeApplications()) { + Set<ApplicationId> applications = activeApplications(); + for (ApplicationId application : applications) { try { // An application might change it's state between the time the set of applications is retrieved and the // time deployment happens. Lock on application and check if it's still active. @@ -53,10 +56,7 @@ public class ApplicationMaintainer extends Maintainer { // deploy asynchronously to make sure we do all applications even when deployments are slow deployAsynchronously(deployment.get()); } - - // Throttle deployments somewhat. - // With a maintenance interval of 30 mins, 1 second is good until 1800 applications - try { Thread.sleep(1000); } catch (InterruptedException e) { return; } + throttle(applications.size()); } catch (RuntimeException e) { log.log(Level.WARNING, "Exception on maintenance redeploy of " + application, e); @@ -74,11 +74,16 @@ public class ApplicationMaintainer extends Maintainer { } }); } + + protected void throttle(int applicationCount) { + // Sleep for a length of time that will spread deployment evenly over the maintenance period + try { Thread.sleep(interval().toMillis() / applicationCount); } catch (InterruptedException e) { return; } + } protected Set<ApplicationId> activeApplications() { return nodeRepository().getNodes(Node.State.active).stream() .map(node -> node.allocation().get().owner()) - .collect(Collectors.toSet()); + .collect(Collectors.toCollection(LinkedHashSet::new)); } private boolean isActive(ApplicationId application) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java index 7ff006bb130..9b363ae9c3f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java @@ -219,7 +219,9 @@ public class ApplicationMaintainerTest { protected void deployAsynchronously(Deployment deployment) { deployment.activate(); } - + + protected void throttle(int applicationCount) { } + @Override protected Set<ApplicationId> activeApplications() { if (overriddenActiveApplications.isPresent()) |