Merge branch 'master' into bratseth/node-metrics

5 files changed, 35 insertions, 17 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
index 3392569d1f2..4d04409aaf0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
@@ -25,15 +25,20 @@ public class InfrastructureProvisioner extends Maintainer {
         this.infraDeployer = infraDeployer;
     }
 
+    public void maintainButThrowOnException() {
+        try {
+            infraDeployer.activateAllSupportedInfraApplications(true);
+        } catch (RuntimeException e) {
+            logger.log(LogLevel.INFO, "Failed to deploy supported infrastructure applications, " +
+                    "will sleep 30s before propagating failure, to allow inspection of zk",
+                    e.getMessage());
+            try { Thread.sleep(30_000); } catch (InterruptedException ignored) { }
+            throw e;
+        }
+    }
+
     @Override
     protected void maintain() {
-        infraDeployer.getSupportedInfraDeployments().forEach((application, deployment) -> {
-            try {
-                deployment.activate();
-            } catch (RuntimeException e) {
-                logger.log(LogLevel.INFO, "Failed to activate " + application, e);
-                // loop around to activate the next application
-            }
-        });
+        infraDeployer.activateAllSupportedInfraApplications(false);
     }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index f265feac788..ecc550527fc 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -94,7 +94,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, defaults.autoscalingInterval);
 
         // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now
-        infrastructureProvisioner.maintain();
+        infrastructureProvisioner.maintainButThrowOnException();
     }
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
index a56ace07c82..1086a3a7cd9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
@@ -20,7 +20,6 @@ import com.yahoo.vespa.service.monitor.DuperModelInfraApi;
 import com.yahoo.vespa.service.monitor.InfraApplicationApi;
 
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -51,9 +50,22 @@ public class InfraDeployerImpl implements InfraDeployer {
     }
 
     @Override
-    public Map<ApplicationId, Deployment> getSupportedInfraDeployments() {
-        return duperModel.getSupportedInfraApplications().stream()
-                .collect(Collectors.toMap(InfraApplicationApi::getApplicationId, InfraDeployment::new));
+    public void activateAllSupportedInfraApplications(boolean propagateException) {
+        duperModel.getSupportedInfraApplications().forEach(api -> {
+            var application = api.getApplicationId();
+            var deployment = new InfraDeployment(api);
+            try {
+                deployment.activate();
+            } catch (RuntimeException e) {
+                logger.log(LogLevel.INFO, "Failed to activate " + application, e);
+                if (propagateException) {
+                    throw e;
+                }
+                // loop around to activate the next application
+            }
+        });
+
+        duperModel.infraApplicationsIsNowComplete();
     }
 
     private class InfraDeployment implements Deployment {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
index ef3d1995df9..e7ebf049e51 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
@@ -50,4 +50,8 @@ public class MockDuperModel implements DuperModelInfraApi {
     public void infraApplicationRemoved(ApplicationId applicationId) {
         activeApps.remove(applicationId);
     }
+
+    @Override
+    public void infraApplicationsIsNowComplete() {
+    }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
index e7bf76986ca..742a863fb38 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
@@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployment;
 import com.yahoo.config.provision.InfraDeployer;
 
-import java.util.Map;
 import java.util.Optional;
 
 public class MockInfraDeployer implements InfraDeployer {
@@ -15,7 +14,5 @@ public class MockInfraDeployer implements InfraDeployer {
     }
 
     @Override
-    public Map<ApplicationId, Deployment> getSupportedInfraDeployments() {
-        return Map.of();
-    }
+    public void activateAllSupportedInfraApplications(boolean propagateException) { }
 }