diff options
author | Harald Musum <musum@yahoo-inc.com> | 2017-03-24 14:47:00 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-03-24 14:47:00 +0100 |
commit | 8844ccb7297e8a5120dd903c85e923f2f93aa693 (patch) | |
tree | 465ea5b86966e1a9c81b40a7fc7945312dabb97b | |
parent | ea3d2e92f60efbfe9f25986066e4048e73a04209 (diff) | |
parent | 8f28065244baefad08b8d9894c67c29f16b14354 (diff) |
Merge pull request #2080 from yahoo/bratseth/smooth-deployments
Spread deployments evently over the maintenance period
2 files changed, 14 insertions, 7 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 72c90aed458..28d6297ca18 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -9,6 +9,8 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.Executor; @@ -39,7 +41,8 @@ public class ApplicationMaintainer extends Maintainer { @Override protected void maintain() { - for (ApplicationId application : activeApplications()) { + Set<ApplicationId> applications = activeApplications(); + for (ApplicationId application : applications) { try { // An application might change it's state between the time the set of applications is retrieved and the // time deployment happens. Lock on application and check if it's still active. @@ -53,10 +56,7 @@ public class ApplicationMaintainer extends Maintainer { // deploy asynchronously to make sure we do all applications even when deployments are slow deployAsynchronously(deployment.get()); } - - // Throttle deployments somewhat. - // With a maintenance interval of 30 mins, 1 second is good until 1800 applications - try { Thread.sleep(1000); } catch (InterruptedException e) { return; } + throttle(applications.size()); } catch (RuntimeException e) { log.log(Level.WARNING, "Exception on maintenance redeploy of " + application, e); @@ -74,11 +74,16 @@ public class ApplicationMaintainer extends Maintainer { } }); } + + protected void throttle(int applicationCount) { + // Sleep for a length of time that will spread deployment evenly over the maintenance period + try { Thread.sleep(interval().toMillis() / applicationCount); } catch (InterruptedException e) { return; } + } protected Set<ApplicationId> activeApplications() { return nodeRepository().getNodes(Node.State.active).stream() .map(node -> node.allocation().get().owner()) - .collect(Collectors.toSet()); + .collect(Collectors.toCollection(LinkedHashSet::new)); } private boolean isActive(ApplicationId application) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java index 7ff006bb130..9b363ae9c3f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java @@ -219,7 +219,9 @@ public class ApplicationMaintainerTest { protected void deployAsynchronously(Deployment deployment) { deployment.activate(); } - + + protected void throttle(int applicationCount) { } + @Override protected Set<ApplicationId> activeApplications() { if (overriddenActiveApplications.isPresent()) |