diff options
author | Jon Bratseth <bratseth@oath.com> | 2021-01-25 23:17:09 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-01-25 23:17:09 +0100 |
commit | d76296e7e32dd85398e8cdf99047cb47a55ba83e (patch) | |
tree | 792d35af0e2358af666f60f1f25b11f22552d8d8 | |
parent | 5410d33fd8146623e2b380dcfe55530b232df336 (diff) | |
parent | 2eaa08b518a984aa3cb2854714644f7e5a6f9545 (diff) |
Merge pull request #16222 from vespa-engine/hmusum/retry-verifyHosts
Retry when verifying hosts [run-systemtest]
-rw-r--r-- | configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java | 25 |
1 files changed, 22 insertions, 3 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 7606aacff15..f5cf7f0dc70 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -34,6 +34,8 @@ import com.yahoo.vespa.curator.Curator; import java.io.File; import java.io.IOException; +import java.time.Duration; +import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -153,9 +155,26 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P } private void validateModelHosts(HostValidator<ApplicationId> hostValidator, ApplicationId applicationId, Model model) { - hostValidator.verifyHosts(applicationId, model.getHosts().stream() - .map(HostInfo::getHostname) - .collect(Collectors.toList())); + // Will retry here, since hosts used might not be in sync on all config servers (we wait for 2/3 servers + // to respond to deployments and deletions). + Instant end = Instant.now().plus(Duration.ofSeconds(1)); + IllegalArgumentException exception; + do { + try { + hostValidator.verifyHosts(applicationId, model.getHosts().stream() + .map(HostInfo::getHostname) + .collect(Collectors.toList())); + return; + } catch (IllegalArgumentException e) { + exception = e; + log.log(Level.INFO, "Verifying hosts failed, will retry: " + e.getMessage()); + try { + Thread.sleep(100); + } catch (InterruptedException interruptedException) {/* ignore */} + } + } while (Instant.now().isBefore(end)); + + throw exception; } /** The result of preparing a single model version */ |