diff options
6 files changed, 29 insertions, 12 deletions
diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index c37e3693457..f2ae997a164 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -506,7 +506,7 @@ ], "methods": [ "public abstract java.util.Optional getDeployment(com.yahoo.config.provision.ApplicationId)", - "public abstract void activateAllSupportedInfraApplications()" + "public abstract void activateAllSupportedInfraApplications(boolean)" ], "fields": [] }, diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java index fa2ee3a90c3..363732ee8a7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import java.util.Map; import java.util.Optional; /** @@ -18,5 +17,5 @@ public interface InfraDeployer { Optional<Deployment> getDeployment(ApplicationId application); /** Deploys all supported infrastructure applications in this zone. */ - void activateAllSupportedInfraApplications(); + void activateAllSupportedInfraApplications(boolean propagateException); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index 51612e27518..cc352cc567f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.InfraDeployer; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; @@ -22,10 +23,22 @@ public class InfrastructureProvisioner extends Maintainer { InfrastructureProvisioner(NodeRepository nodeRepository, InfraDeployer infraDeployer, Duration interval) { super(nodeRepository, interval); this.infraDeployer = infraDeployer; + + // If this fails, we fail the component graph construction and bootstrap of config server, + // which is what we want. + try { + infraDeployer.activateAllSupportedInfraApplications(true); + } catch (RuntimeException e) { + logger.log(LogLevel.INFO, "Failed to deploy supported infrastructure applications, " + + "will sleep 30s before propagating failure, to allow inspection of zk", + e.getMessage()); + try { Thread.sleep(30_000); } catch (InterruptedException ignored) { } + throw e; + } } @Override protected void maintain() { - infraDeployer.activateAllSupportedInfraApplications(); + infraDeployer.activateAllSupportedInfraApplications(false); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index be42d3aed80..1086a3a7cd9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -20,7 +20,6 @@ import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.InfraApplicationApi; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -51,7 +50,7 @@ public class InfraDeployerImpl implements InfraDeployer { } @Override - public void activateAllSupportedInfraApplications() { + public void activateAllSupportedInfraApplications(boolean propagateException) { duperModel.getSupportedInfraApplications().forEach(api -> { var application = api.getApplicationId(); var deployment = new InfraDeployment(api); @@ -59,6 +58,9 @@ public class InfraDeployerImpl implements InfraDeployer { deployment.activate(); } catch (RuntimeException e) { logger.log(LogLevel.INFO, "Failed to activate " + application, e); + if (propagateException) { + throw e; + } // loop around to activate the next application } }); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java index 45fb2742bd4..742a863fb38 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.InfraDeployer; -import java.util.Map; import java.util.Optional; public class MockInfraDeployer implements InfraDeployer { @@ -15,5 +14,5 @@ public class MockInfraDeployer implements InfraDeployer { } @Override - public void activateAllSupportedInfraApplications() { } + public void activateAllSupportedInfraApplications(boolean propagateException) { } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index 60e4c30a634..15c461c7f59 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -94,8 +94,10 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi @Override public void notifyOfCompleteness(SuperModel superModel) { synchronized (monitor) { - superModelIsComplete = true; - maybeSetDuperModelAsComplete(); + if (!superModelIsComplete) { + superModelIsComplete = true; + maybeSetDuperModelAsComplete(); + } } } }); @@ -164,8 +166,10 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi @Override public void infraApplicationsIsNowComplete() { synchronized (monitor) { - this.infraApplicationsIsComplete = true; - maybeSetDuperModelAsComplete(); + if (!infraApplicationsIsComplete) { + infraApplicationsIsComplete = true; + maybeSetDuperModelAsComplete(); + } } } |