diff options
author | jonmv <venstad@gmail.com> | 2022-09-26 11:56:08 +0200 |
---|---|---|
committer | jonmv <venstad@gmail.com> | 2022-09-26 11:56:08 +0200 |
commit | a76b01d99f10ae5c01ccf4ca40479a093b35a231 (patch) | |
tree | 4ec9852619715a65ecb9eaeca7fd3b7efca13a7f /controller-server | |
parent | 702597f2d08121b93e6768d221b84bf36b1834ce (diff) |
Defer upgrades of active deployments in perf for up to one week
Diffstat (limited to 'controller-server')
-rw-r--r-- | controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java | 15 |
1 files changed, 14 insertions, 1 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index fd177c469d2..934a1b4fa2f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; +import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -11,11 +12,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; +import java.util.Comparator; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; @@ -37,7 +40,8 @@ public class DeploymentUpgrader extends ControllerMaintainer { AtomicInteger failures = new AtomicInteger(); Version targetPlatform = null; // Upgrade to the newest non-broken, deployable version. - for (VespaVersion platform : controller().readVersionStatus().deployableVersions()) + VersionStatus versionStatus = controller().readVersionStatus(); + for (VespaVersion platform : versionStatus.deployableVersions()) if (platform.confidence().equalOrHigherThan(VespaVersion.Confidence.normal)) targetPlatform = platform.versionNumber(); @@ -63,6 +67,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { if ( ! deployment.version().isBefore(target.targetPlatform())) continue; if ( ! isLikelyNightFor(job)) continue; + if (deployment.zone().environment() == Environment.perf && ! isIdleOrOutdated(deployment, job)) continue; log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); attempts.incrementAndGet(); @@ -76,6 +81,14 @@ public class DeploymentUpgrader extends ControllerMaintainer { return asSuccessFactor(attempts.get(), failures.get()); } + /** Returns whether query and feed metrics are ~zero, or currently platform has been deployed for a week. */ + private boolean isIdleOrOutdated(Deployment deployment, JobId job) { + if (deployment.metrics().queriesPerSecond() <= 1 && deployment.metrics().writesPerSecond() <= 1) return true; + return controller().jobController().runs(job).descendingMap().values().stream() + .takeWhile(run -> run.versions().targetPlatform().equals(deployment.version())) + .anyMatch(run -> run.start().isBefore(controller().clock().instant().minus(Duration.ofDays(7)))); + } + private boolean isLikelyNightFor(JobId job) { int hour = hourOf(controller().clock().instant()); int[] runStarts = controller().jobController().jobStarts(job).stream() |