diff options
author | Harald Musum <musum@verizonmedia.com> | 2021-01-25 23:09:22 +0100 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2021-01-25 23:09:22 +0100 |
commit | 2eaa08b518a984aa3cb2854714644f7e5a6f9545 (patch) | |
tree | 669d77c06069e1a799652cacc5dc61adc3752844 | |
parent | aaae06855cd1ba0fb2b684dd683bd07e2e0e1165 (diff) |
Retry when verifying hosts
Hosts used might not be in sync on all config servers (we wait for 2/3
servers to respond to deployments and deletions). Retry verification
of hosts for 1 second
-rw-r--r-- | configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java | 25 |
1 files changed, 22 insertions, 3 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 7606aacff15..f5cf7f0dc70 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -34,6 +34,8 @@ import com.yahoo.vespa.curator.Curator; import java.io.File; import java.io.IOException; +import java.time.Duration; +import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -153,9 +155,26 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P } private void validateModelHosts(HostValidator<ApplicationId> hostValidator, ApplicationId applicationId, Model model) { - hostValidator.verifyHosts(applicationId, model.getHosts().stream() - .map(HostInfo::getHostname) - .collect(Collectors.toList())); + // Will retry here, since hosts used might not be in sync on all config servers (we wait for 2/3 servers + // to respond to deployments and deletions). + Instant end = Instant.now().plus(Duration.ofSeconds(1)); + IllegalArgumentException exception; + do { + try { + hostValidator.verifyHosts(applicationId, model.getHosts().stream() + .map(HostInfo::getHostname) + .collect(Collectors.toList())); + return; + } catch (IllegalArgumentException e) { + exception = e; + log.log(Level.INFO, "Verifying hosts failed, will retry: " + e.getMessage()); + try { + Thread.sleep(100); + } catch (InterruptedException interruptedException) {/* ignore */} + } + } while (Instant.now().isBefore(end)); + + throw exception; } /** The result of preparing a single model version */ |