Merge pull request #24211 from vespa-engine/jonmv/defer-perf-upgrades-while-deployment-is-active

Defer upgrades of active deployments in perf for up to one week

1 files changed, 14 insertions, 1 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
index fd177c469d2..934a1b4fa2f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
@@ -11,11 +12,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.Versions;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Comparator;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
@@ -37,7 +40,8 @@ public class DeploymentUpgrader extends ControllerMaintainer {
         AtomicInteger failures = new AtomicInteger();
 
         Version targetPlatform = null; // Upgrade to the newest non-broken, deployable version.
-        for (VespaVersion platform : controller().readVersionStatus().deployableVersions())
+        VersionStatus versionStatus = controller().readVersionStatus();
+        for (VespaVersion platform : versionStatus.deployableVersions())
             if (platform.confidence().equalOrHigherThan(VespaVersion.Confidence.normal))
                 targetPlatform = platform.versionNumber();
 
@@ -63,6 +67,7 @@ public class DeploymentUpgrader extends ControllerMaintainer {
 
                         if ( ! deployment.version().isBefore(target.targetPlatform())) continue;
                         if ( ! isLikelyNightFor(job)) continue;
+                        if (deployment.zone().environment() == Environment.perf && ! isIdleOrOutdated(deployment, job)) continue;
 
                         log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone());
                         attempts.incrementAndGet();
@@ -76,6 +81,14 @@ public class DeploymentUpgrader extends ControllerMaintainer {
         return asSuccessFactor(attempts.get(), failures.get());
     }
 
+    /** Returns whether query and feed metrics are ~zero, or currently platform has been deployed for a week. */
+    private boolean isIdleOrOutdated(Deployment deployment, JobId job) {
+        if (deployment.metrics().queriesPerSecond() <= 1 && deployment.metrics().writesPerSecond() <= 1) return true;
+        return controller().jobController().runs(job).descendingMap().values().stream()
+                           .takeWhile(run -> run.versions().targetPlatform().equals(deployment.version()))
+                           .anyMatch(run -> run.start().isBefore(controller().clock().instant().minus(Duration.ofDays(7))));
+    }
+
     private boolean isLikelyNightFor(JobId job) {
         int hour = hourOf(controller().clock().instant());
         int[] runStarts = controller().jobController().jobStarts(job).stream()