Deploy the app that was deployed the longest time ago

Run PeriodicApplicationMaintainer more often and deploy the app
that was deployed the longest time ago

4 files changed, 38 insertions, 14 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
index abee3b6ef7f..b9dff572d5b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
@@ -63,7 +63,7 @@ public abstract class ApplicationMaintainer extends Maintainer {
     protected Deployer deployer() { return deployer; }
 
 
-    private Set<ApplicationId> applicationsNeedingMaintenance() {
+    protected Set<ApplicationId> applicationsNeedingMaintenance() {
         return nodesNeedingMaintenance().stream()
                 .map(node -> node.allocation().get().owner())
                 .collect(Collectors.toCollection(LinkedHashSet::new));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 391807ece95..6985206b78e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -71,7 +71,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         infrastructureVersions = new InfrastructureVersions(nodeRepository.database());
 
         nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, durationFromEnv("fail_grace").orElse(defaults.failGrace), clock, orchestrator, throttlePolicyFromEnv("throttle_policy").orElse(defaults.throttlePolicy), metric, jobControl, configserverConfig);
-        periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, nodeRepository, durationFromEnv("periodic_redeploy_interval").orElse(defaults.periodicRedeployInterval), jobControl);
+        periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, nodeRepository, defaults.redeployMaintainerInterval, durationFromEnv("periodic_redeploy_interval").orElse(defaults.periodicRedeployInterval), jobControl);
         operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, nodeRepository, clock, durationFromEnv("operator_change_redeploy_interval").orElse(defaults.operatorChangeRedeployInterval), jobControl);
         reservationExpirer = new ReservationExpirer(nodeRepository, clock, durationFromEnv("reservation_expiry").orElse(defaults.reservationExpiry), jobControl);
         retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, clock, durationFromEnv("retired_interval").orElse(defaults.retiredInterval), durationFromEnv("retired_expiry").orElse(defaults.retiredExpiry), jobControl);
@@ -130,8 +130,11 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
 
     private static class DefaultTimes {
 
-        /** All applications are redeployed with this period */
+        // TODO: Rename, kept now for compatibility reasons, want to change this and corresponding env variable
+        /** Minimum time to wait between deployments by periodic application maintainer*/
         private final Duration periodicRedeployInterval;
+        /** Time between each run of maintainer that does periodic redeployment */
+        private final Duration redeployMaintainerInterval;
         /** Applications are redeployed after manual operator changes within this time period */
         private final Duration operatorChangeRedeployInterval;
 
@@ -155,6 +158,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         DefaultTimes(Zone zone) {
             failGrace = Duration.ofMinutes(60);
             periodicRedeployInterval = Duration.ofMinutes(30);
+            redeployMaintainerInterval = Duration.ofMinutes(1);
             operatorChangeRedeployInterval = Duration.ofMinutes(1);
             failedExpirerInterval = Duration.ofMinutes(10);
             provisionedExpiry = Duration.ofHours(4);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
index e682f2dc76c..2ce5046edbf 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
@@ -8,8 +8,12 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 
 /**
  * The application maintainer regularly redeploys all applications to make sure the node repo and application
@@ -19,19 +23,31 @@ import java.util.Optional;
  * @author bratseth
  */
 public class PeriodicApplicationMaintainer extends ApplicationMaintainer {
+    private final Duration minTimeBetweenRedeployments;
 
     public PeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, 
-                                         Duration interval, JobControl jobControl) {
+                                         Duration interval, Duration minTimeBetweenRedeployments, JobControl jobControl) {
         super(deployer, nodeRepository, interval, jobControl);
+        this.minTimeBetweenRedeployments = minTimeBetweenRedeployments;
     }
 
     @Override
     protected boolean canDeployNow(ApplicationId application) {
-        Optional<Instant> lastDeploy = deployer().lastDeployTime(application);
-        if (lastDeploy.isPresent() &&
-            lastDeploy.get().isAfter(nodeRepository().clock().instant().minus(interval())))
-            return false; // Don't deploy if a regular deploy just happened
-        return true;
+        // Don't deploy if a regular deploy just happened
+        return getLastDeployTime(application).isBefore(nodeRepository().clock().instant().minus(minTimeBetweenRedeployments));
+    }
+
+    // Returns the app that was deployed the longest time ago
+    @Override
+    protected Set<ApplicationId> applicationsNeedingMaintenance() {
+        Optional<ApplicationId> apps = (nodesNeedingMaintenance().stream()
+                .map(node -> node.allocation().get().owner())
+                .min(Comparator.comparing(this::getLastDeployTime)));
+        return apps.map(applicationId -> new HashSet<>(Collections.singletonList(applicationId))).orElseGet(HashSet::new);
+    }
+
+    private Instant getLastDeployTime(ApplicationId application) {
+        return deployer().lastDeployTime(application).orElse(Instant.EPOCH);
     }
 
     @Override
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index edbb965772e..90fa1801789 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -32,7 +32,6 @@ import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -146,7 +145,7 @@ public class PeriodicApplicationMaintainerTest {
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get());
         ((ManualClock) nodeRepository.clock()).advance(Duration.ofMinutes(5));
         fixture.runApplicationMaintainer();
-        // Too soo: Not redeployed:
+        // Too soon: Not redeployed:
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app1).get());
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get());
 
@@ -227,7 +226,12 @@ public class PeriodicApplicationMaintainerTest {
         }
 
         void runApplicationMaintainer(Optional<List<Node>> overriddenNodesNeedingMaintenance) {
-            new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofMinutes(30), overriddenNodesNeedingMaintenance).run();
+            TestablePeriodicApplicationMaintainer maintainer =
+                    new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofMinutes(1),
+                                                              Duration.ofMinutes(30), overriddenNodesNeedingMaintenance);
+            // Need to run twice, as only one app is deployed per run
+            maintainer.run();
+            maintainer.run();
         }
 
         NodeList getNodes(Node.State ... states) {
@@ -241,8 +245,8 @@ public class PeriodicApplicationMaintainerTest {
         private Optional<List<Node>> overriddenNodesNeedingMaintenance;
         
         TestablePeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval,
-                                              Optional<List<Node>> overriddenNodesNeedingMaintenance) {
-            super(deployer, nodeRepository, interval, new JobControl(nodeRepository.database()));
+                                              Duration minTimeBetweenRedeployments, Optional<List<Node>> overriddenNodesNeedingMaintenance) {
+            super(deployer, nodeRepository, interval, minTimeBetweenRedeployments, new JobControl(nodeRepository.database()));
             this.overriddenNodesNeedingMaintenance = overriddenNodesNeedingMaintenance;
         }