From 2eaa08b518a984aa3cb2854714644f7e5a6f9545 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 25 Jan 2021 23:09:22 +0100 Subject: Retry when verifying hosts Hosts used might not be in sync on all config servers (we wait for 2/3 servers to respond to deployments and deletions). Retry verification of hosts for 1 second --- .../server/modelfactory/PreparedModelsBuilder.java | 25 +++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 7606aacff15..f5cf7f0dc70 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -34,6 +34,8 @@ import com.yahoo.vespa.curator.Curator; import java.io.File; import java.io.IOException; +import java.time.Duration; +import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -153,9 +155,26 @@ public class PreparedModelsBuilder extends ModelsBuilder hostValidator, ApplicationId applicationId, Model model) { - hostValidator.verifyHosts(applicationId, model.getHosts().stream() - .map(HostInfo::getHostname) - .collect(Collectors.toList())); + // Will retry here, since hosts used might not be in sync on all config servers (we wait for 2/3 servers + // to respond to deployments and deletions). + Instant end = Instant.now().plus(Duration.ofSeconds(1)); + IllegalArgumentException exception; + do { + try { + hostValidator.verifyHosts(applicationId, model.getHosts().stream() + .map(HostInfo::getHostname) + .collect(Collectors.toList())); + return; + } catch (IllegalArgumentException e) { + exception = e; + log.log(Level.INFO, "Verifying hosts failed, will retry: " + e.getMessage()); + try { + Thread.sleep(100); + } catch (InterruptedException interruptedException) {/* ignore */} + } + } while (Instant.now().isBefore(end)); + + throw exception; } /** The result of preparing a single model version */ -- cgit v1.2.3