Merge pull request #20832 from vespa-engine/hmusum/wait-for-config-generation-before-restarting

Hmusum/wait for config generation before restarting [run-systemtest]

13 files changed, 197 insertions, 44 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 63ab00275bf..2ee3e6c0343 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -88,7 +88,6 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
-import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.time.Clock;
@@ -111,8 +110,8 @@ import java.util.stream.Collectors;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER;
 import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER;
-import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse;
 import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse;
+import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse;
 import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk;
 import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk;
 import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT;
@@ -147,6 +146,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final Metric metric;
     private final SecretStoreValidator secretStoreValidator;
     private final ClusterReindexingStatusClient clusterReindexingStatusClient;
+    private final FlagSource flagSource;
 
     @Inject
     public ApplicationRepository(TenantRepository tenantRepository,
@@ -203,8 +203,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.metric = Objects.requireNonNull(metric);
         this.secretStoreValidator = Objects.requireNonNull(secretStoreValidator);
         this.clusterReindexingStatusClient = clusterReindexingStatusClient;
+        this.flagSource = flagSource;
     }
 
+    // Should be used by tests only (first constructor in this class makes sure we use injectable components where possible)
     public static class Builder {
         private TenantRepository tenantRepository;
         private Optional<Provisioner> hostProvisioner;
@@ -217,6 +219,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         private Metric metric = new NullMetric();
         private SecretStoreValidator secretStoreValidator = new SecretStoreValidator(new SecretStoreProvider().get());
         private FlagSource flagSource = new InMemoryFlagSource();
+        private ConfigConvergenceChecker configConvergenceChecker = new ConfigConvergenceChecker();
 
         public Builder withTenantRepository(TenantRepository tenantRepository) {
             this.tenantRepository = tenantRepository;
@@ -280,11 +283,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
             return this;
         }
 
+        public Builder withConfigConvergenceChecker(ConfigConvergenceChecker configConvergenceChecker) {
+            this.configConvergenceChecker = configConvergenceChecker;
+            return this;
+        }
+
         public ApplicationRepository build() {
             return new ApplicationRepository(tenantRepository,
                                              hostProvisioner,
                                              InfraDeployerProvider.empty().getInfraDeployer(),
-                                             new ConfigConvergenceChecker(),
+                                             configConvergenceChecker,
                                              httpProxy,
                                              configserverConfig,
                                              orchestrator,
@@ -747,10 +755,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     }
 
     public ServiceListResponse servicesToCheckForConfigConvergence(ApplicationId applicationId,
-                                                                   URI uri,
                                                                    Duration timeoutPerService,
                                                                    Optional<Version> vespaVersion) {
-        return convergeChecker.getServiceConfigGenerations(getApplication(applicationId, vespaVersion), uri, timeoutPerService);
+        return convergeChecker.getConfigGenerationsForAllServices(getApplication(applicationId, vespaVersion), timeoutPerService);
     }
 
     public ConfigConvergenceChecker configConvergenceChecker() { return convergeChecker; }
@@ -1016,6 +1023,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return applicationId.orElse(null);
     }
 
+    public FlagSource flagSource() { return flagSource; }
+
     private Session validateThatLocalSessionIsNotActive(Tenant tenant, long sessionId) {
         Session session = getLocalSession(tenant, sessionId);
         if (Session.Status.ACTIVATE.equals(session.getStatus())) {
@@ -1042,6 +1051,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return getTenant(appId).getSessionRepository().getActiveApplicationSet(appId);
     }
 
+    public Application getActiveApplication(ApplicationId applicationId) {
+        return getActiveApplicationSet(applicationId)
+                .map(a -> a.getForVersionOrLatest(Optional.empty(), clock.instant()))
+                .orElseThrow(() -> new RuntimeException("Found no active application for " + applicationId));
+    }
+
     private File decompressApplication(InputStream in, String contentType, File tempDir) {
         try (CompressedApplicationInputStream application =
                      CompressedApplicationInputStream.createFromCompressedStream(in, contentType)) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java
index ad14cf4aab6..d8dc26b0b3a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java
@@ -39,8 +39,9 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -71,7 +72,7 @@ public class ConfigConvergenceChecker extends AbstractComponent {
     );
 
 
-    private final Executor responseHandlerExecutor =
+    private final ExecutorService responseHandlerExecutor =
             Executors.newSingleThreadExecutor(new DaemonThreadFactory("config-convergence-checker-response-handler-"));
     private final ObjectMapper jsonMapper = new ObjectMapper();
 
@@ -90,10 +91,10 @@ public class ConfigConvergenceChecker extends AbstractComponent {
     }
 
     /** Check all services in given application. Returns the minimum current generation of all services */
-    public ServiceListResponse getServiceConfigGenerations(Application application, URI uri, Duration timeoutPerService) {
+    public ServiceListResponse getConfigGenerationsForAllServices(Application application, Duration timeoutPerService) {
         Map<ServiceInfo, Long> currentGenerations = getServiceConfigGenerations(application, timeoutPerService);
         long currentGeneration = currentGenerations.values().stream().mapToLong(Long::longValue).min().orElse(-1);
-        return new ServiceListResponse(currentGenerations, uri, application.getApplicationGeneration(), currentGeneration);
+        return new ServiceListResponse(currentGenerations, application.getApplicationGeneration(), currentGeneration);
     }
 
     /** Check service identified by host and port in given application */
@@ -196,6 +197,16 @@ public class ConfigConvergenceChecker extends AbstractComponent {
                       .findFirst();
     }
 
+    @Override
+    public void deconstruct()  {
+        responseHandlerExecutor.shutdown();
+        try {
+            responseHandlerExecutor.awaitTermination(10, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            log.log(Level.WARNING, "Unable to shutdown executor", e);
+        }
+    }
+
     private static long generationFromContainerState(JsonNode state) {
         return state.get("config").get("generation").asLong(-1);
     }
@@ -256,23 +267,23 @@ public class ConfigConvergenceChecker extends AbstractComponent {
         public final boolean converged;
         public final Optional<String> errorMessage;
 
-        public ServiceResponse(Status status, Long wantedGeneration) {
-            this(status, wantedGeneration, 0L);
+        public ServiceResponse(Status status, long wantedGeneration) {
+            this(status, wantedGeneration, 0);
         }
 
-        public ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration) {
+        public ServiceResponse(Status status, long wantedGeneration, long currentGeneration) {
             this(status, wantedGeneration, currentGeneration, false);
         }
 
-        public ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration, boolean converged) {
+        public ServiceResponse(Status status, long wantedGeneration, long currentGeneration, boolean converged) {
             this(status, wantedGeneration, currentGeneration, converged, Optional.empty());
         }
 
-        public ServiceResponse(Status status, Long wantedGeneration, String errorMessage) {
-            this(status, wantedGeneration, 0L, false, Optional.ofNullable(errorMessage));
+        public ServiceResponse(Status status, long wantedGeneration, String errorMessage) {
+            this(status, wantedGeneration, 0, false, Optional.ofNullable(errorMessage));
         }
 
-        private ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration, boolean converged, Optional<String> errorMessage) {
+        private ServiceResponse(Status status, long wantedGeneration, long currentGeneration, boolean converged, Optional<String> errorMessage) {
             this.status = status;
             this.wantedGeneration = wantedGeneration;
             this.currentGeneration = currentGeneration;
@@ -285,15 +296,15 @@ public class ConfigConvergenceChecker extends AbstractComponent {
     public static class ServiceListResponse {
 
         public final List<Service> services = new ArrayList<>();
-        public final URI uri;
         public final long wantedGeneration;
         public final long currentGeneration;
+        public final boolean converged;
 
-        public ServiceListResponse(Map<ServiceInfo, Long> services, URI uri, long wantedGeneration, long currentGeneration) {
+        public ServiceListResponse(Map<ServiceInfo, Long> services, long wantedGeneration, long currentGeneration) {
             services.forEach((key, value) -> this.services.add(new Service(key, value)));
-            this.uri = uri;
             this.wantedGeneration = wantedGeneration;
             this.currentGeneration = currentGeneration;
+            this.converged = currentGeneration >= wantedGeneration;
         }
 
         public List<Service> services() { return services; }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java
index f0711e5c238..88cddb93d9d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java
@@ -5,7 +5,13 @@ package com.yahoo.vespa.config.server.application;
  * @author Ulf Lilleengen
  */
 public class ConfigNotConvergedException extends RuntimeException {
+
+    public ConfigNotConvergedException(Throwable t) {
+        super(t);
+    }
+
     public ConfigNotConvergedException(String message) {
         super(message);
     }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
index 4b7e39f48a4..49dade6f0cd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.deploy;
 
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
+import com.yahoo.concurrent.UncheckedTimeoutException;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.api.ServiceInfo;
@@ -19,12 +20,17 @@ import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer;
 import com.yahoo.vespa.config.server.ApplicationRepository.Activation;
 import com.yahoo.vespa.config.server.TimeoutBudget;
+import com.yahoo.vespa.config.server.application.Application;
+import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker;
+import com.yahoo.vespa.config.server.application.ConfigNotConvergedException;
 import com.yahoo.vespa.config.server.configchange.ConfigChangeActions;
 import com.yahoo.vespa.config.server.configchange.ReindexActions;
 import com.yahoo.vespa.config.server.configchange.RestartActions;
 import com.yahoo.vespa.config.server.session.PrepareParams;
 import com.yahoo.vespa.config.server.session.Session;
 import com.yahoo.vespa.config.server.tenant.Tenant;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.Flags;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -35,6 +41,8 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
+import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse;
+
 /**
  * The process of deploying an application.
  * Deployments are created by an {@link ApplicationRepository}.
@@ -149,6 +157,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
         RestartActions restartActions = configChangeActions.getRestartActions().useForInternalRestart(internalRedeploy);
 
         if ( ! restartActions.isEmpty()) {
+
+            waitForConfigToConverge(applicationId);
+
             Set<String> hostnames = restartActions.getEntries().stream()
                                                   .flatMap(entry -> entry.getServices().stream())
                                                   .map(ServiceInfo::getHostName)
@@ -165,6 +176,39 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
         }
     }
 
+    private void waitForConfigToConverge(ApplicationId applicationId) {
+        BooleanFlag verify = Flags.CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING.bindTo(applicationRepository.flagSource());
+        if ( ! verify.value()) return;
+
+        // Timeout per service when getting config generations
+        Duration timeout = Duration.ofSeconds(10);
+        while (true) {
+            try {
+                params.get().getTimeoutBudget().assertNotTimedOut(
+                        () -> "Timeout exceeded while waiting for config convergence for " + applicationId);
+            } catch (UncheckedTimeoutException e) {
+                throw new ConfigNotConvergedException(e);
+            }
+
+            Application app = applicationRepository.getActiveApplication(applicationId);
+
+            // TODO: Don't wait for config convergence if restartOnDeploy is true for one o more container clusters
+            // (ideally wait for config convergence for all other services)
+
+            log.info(session.logPre() + "Wait for services to converge on new generation before restarting");
+            ConfigConvergenceChecker convergenceChecker = applicationRepository.configConvergenceChecker();
+            ServiceListResponse response = convergenceChecker.getConfigGenerationsForAllServices(app, timeout);
+            if (response.converged) {
+                log.info(session.logPre() + "Services converged on new generation " + response.currentGeneration);
+                return;
+            } else {
+                log.info(session.logPre() + "Services not converged on new generation, wanted generation: " + response.wantedGeneration +
+                                 ", current generation: " + response.currentGeneration + ", will retry");
+                try { Thread.sleep(10_000); } catch (InterruptedException e) { /* ignore */ }
+            }
+        }
+    }
+
     private void storeReindexing(ApplicationId applicationId, long requiredSession) {
         applicationRepository.modifyReindexing(applicationId, reindexing -> {
             if (configChangeActions != null)
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java
index 96e1767b462..c414d8dfa9a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java
@@ -2,13 +2,13 @@
 package com.yahoo.vespa.config.server.http;
 
 import com.yahoo.container.jdisc.HttpResponse;
-import java.util.logging.Level;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.JsonFormat;
 import com.yahoo.slime.Slime;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
@@ -50,7 +50,8 @@ public class HttpErrorResponse extends HttpResponse {
         UNKNOWN_VESPA_VERSION,
         PARENT_HOST_NOT_READY,
         CERTIFICATE_NOT_READY,
-        LOAD_BALANCER_NOT_READY
+        LOAD_BALANCER_NOT_READY,
+        CONFIG_NOT_CONVERGED
     }
 
     public static HttpErrorResponse notFoundError(String msg) {
@@ -101,6 +102,10 @@ public class HttpErrorResponse extends HttpResponse {
         return new HttpErrorResponse(CONFLICT, ErrorCode.CERTIFICATE_NOT_READY.name(), msg);
     }
 
+    public static HttpErrorResponse configNotConverged(String msg) {
+        return new HttpErrorResponse(CONFLICT, ErrorCode.CONFIG_NOT_CONVERGED.name(), msg);
+    }
+
     public static HttpErrorResponse loadBalancerNotReady(String msg) {
         return new HttpErrorResponse(CONFLICT, ErrorCode.LOAD_BALANCER_NOT_READY.name(), msg);
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java
index 2dad2c060cc..9f4f8b5f9ce 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java
@@ -10,6 +10,7 @@ import com.yahoo.config.provision.exception.LoadBalancerServiceException;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.vespa.config.server.application.ConfigNotConvergedException;
 import com.yahoo.yolean.Exceptions;
 
 import java.io.PrintWriter;
@@ -68,6 +69,8 @@ public class HttpHandler extends LoggingRequestHandler {
             return HttpErrorResponse.parentHostNotReady(getMessage(e, request));
         } catch (CertificateNotReadyException e) {
             return HttpErrorResponse.certificateNotReady(getMessage(e, request));
+        } catch (ConfigNotConvergedException e) {
+            return HttpErrorResponse.configNotConverged(getMessage(e, request));
         } catch (LoadBalancerServiceException e) {
             return HttpErrorResponse.loadBalancerNotReady(getMessage(e, request));
         } catch (Exception e) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 0131517818d..df4df234ed0 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -117,10 +117,9 @@ public class ApplicationHandler extends HttpHandler {
     private HttpResponse listServiceConverge(ApplicationId applicationId, HttpRequest request) {
         ServiceListResponse response =
                 applicationRepository.servicesToCheckForConfigConvergence(applicationId,
-                                                                          request.getUri(),
                                                                           getTimeoutFromRequest(request),
                                                                           getVespaVersionFromRequest(request));
-        return new HttpServiceListResponse(response);
+        return new HttpServiceListResponse(response, request.getUri());
     }
 
     private HttpResponse checkServiceConverge(ApplicationId applicationId, String hostAndPort, HttpRequest request) {
@@ -370,7 +369,7 @@ public class ApplicationHandler extends HttpHandler {
     static class HttpServiceListResponse extends JSONResponse {
 
         // Pre-condition: servicesToCheck has a state port
-        public HttpServiceListResponse(ConfigConvergenceChecker.ServiceListResponse response) {
+        public HttpServiceListResponse(ConfigConvergenceChecker.ServiceListResponse response, URI uri) {
             super(200);
             Cursor serviceArray = object.setArray("services");
             response.services().forEach((service) -> {
@@ -381,13 +380,13 @@ public class ApplicationHandler extends HttpHandler {
                 serviceObject.setString("host", hostName);
                 serviceObject.setLong("port", statePort);
                 serviceObject.setString("type", serviceInfo.getServiceType());
-                serviceObject.setString("url", response.uri.toString() + "/" + hostName + ":" + statePort);
+                serviceObject.setString("url", uri.toString() + "/" + hostName + ":" + statePort);
                 serviceObject.setLong("currentGeneration", service.currentGeneration);
             });
-            object.setString("url", response.uri.toString());
+            object.setString("url", uri.toString());
             object.setLong("currentGeneration", response.currentGeneration);
             object.setLong("wantedGeneration", response.wantedGeneration);
-            object.setBool("converged", response.currentGeneration >= response.wantedGeneration);
+            object.setBool("converged", response.converged);
         }
     }
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index c2af7f1ee1f..284a0aa0a62 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -103,6 +103,7 @@ public class ApplicationRepositoryTest {
     private OrchestratorMock orchestrator;
     private TimeoutBudget timeoutBudget;
     private Curator curator;
+    private ConfigserverConfig configserverConfig;
 
     @Rule
     public TemporaryFolder temporaryFolder = new TemporaryFolder();
@@ -114,7 +115,7 @@ public class ApplicationRepositoryTest {
     @Before
     public void setup() throws IOException {
         curator = new MockCurator();
-        ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder()
+        configserverConfig = new ConfigserverConfig.Builder()
                 .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED)
                 .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath())
                 .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath())
@@ -170,6 +171,16 @@ public class ApplicationRepositoryTest {
 
     @Test
     public void prepareAndActivateWithRestart() {
+        applicationRepository = new ApplicationRepository.Builder()
+                .withTenantRepository(tenantRepository)
+                .withProvisioner(provisioner)
+                .withConfigserverConfig(configserverConfig)
+                .withOrchestrator(orchestrator)
+                .withLogRetriever(new MockLogRetriever())
+                .withClock(clock)
+                .withConfigConvergenceChecker(new MockConfigConvergenceChecker(2))
+                .build();
+
         prepareAndActivate(testAppJdiscOnly);
         PrepareResult result = prepareAndActivate(testAppJdiscOnlyRestart);
         assertTrue(result.configChangeActions().getRefeedActions().isEmpty());
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java
new file mode 100644
index 00000000000..b21d89fa626
--- /dev/null
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java
@@ -0,0 +1,33 @@
+package com.yahoo.vespa.config.server;
+
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.vespa.config.server.application.Application;
+import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker;
+
+import java.time.Duration;
+import java.util.Map;
+
+public class MockConfigConvergenceChecker extends ConfigConvergenceChecker {
+
+    private final long wantedGeneration;
+
+    public MockConfigConvergenceChecker(long wantedGeneration) {
+        this.wantedGeneration = wantedGeneration;
+    }
+
+    @Override
+    public Map<ServiceInfo, Long> getServiceConfigGenerations(Application application, Duration timeoutPerService) {
+        return Map.of();
+    }
+
+    @Override
+    public ServiceListResponse getConfigGenerationsForAllServices(Application application, Duration timeoutPerService) {
+        return new ServiceListResponse(Map.of(), wantedGeneration, wantedGeneration);
+    }
+
+    @Override
+    public ServiceResponse getServiceConfigGeneration(Application application, String hostAndPortToCheck, Duration timeout) {
+        return new ServiceResponse(ServiceResponse.Status.ok, wantedGeneration);
+    }
+
+}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java
index 6afb9ef086d..148cd8ac24f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java
@@ -28,6 +28,7 @@ import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options
 import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse;
 import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -88,12 +89,12 @@ public class ConfigConvergenceCheckerTest {
     @Test
     public void service_list_convergence() {
         {
-            URI requestUrl = testServer().resolve("/serviceconverge");
             wireMock.stubFor(get(urlEqualTo("/state/v1/config")).willReturn(okJson("{\"config\":{\"generation\":3}}")));
 
-            ServiceListResponse response = checker.getServiceConfigGenerations(application, requestUrl, clientTimeout);
+            ServiceListResponse response = checker.getConfigGenerationsForAllServices(application, clientTimeout);
             assertEquals(3, response.wantedGeneration);
             assertEquals(3, response.currentGeneration);
+            assertTrue(response.converged);
             List<ServiceListResponse.Service> services = response.services;
             assertEquals(1, services.size());
             assertService(this.service, services.get(0), 3);
@@ -113,9 +114,10 @@ public class ConfigConvergenceCheckerTest {
 
             URI requestUrl = testServer().resolve("/serviceconverge");
 
-            ServiceListResponse response = checker.getServiceConfigGenerations(application, requestUrl, clientTimeout);
+            ServiceListResponse response = checker.getConfigGenerationsForAllServices(application, clientTimeout);
             assertEquals(4, response.wantedGeneration);
             assertEquals(3, response.currentGeneration);
+            assertFalse(response.converged);
 
             List<ServiceListResponse.Service> services = response.services;
             assertEquals(2, services.size());
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
index f16b1102f2a..7f223fbce6f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
@@ -24,6 +24,7 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.MockProvisioner;
 import com.yahoo.vespa.config.server.TimeoutBudget;
+import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker;
 import com.yahoo.vespa.config.server.application.OrchestratorMock;
 import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory;
 import com.yahoo.vespa.config.server.http.v2.PrepareResult;
@@ -37,9 +38,10 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository;
 import com.yahoo.vespa.config.server.tenant.TestTenantRepository;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.VespaModelFactory;
-import com.yahoo.vespa.orchestrator.Orchestrator;
 
 import java.io.File;
 import java.nio.file.Files;
@@ -253,7 +255,8 @@ public class DeployTester {
         private Curator curator = new MockCurator();
         private Metrics metrics;
         private List<ModelFactory> modelFactories;
-        private Orchestrator orchestrator;
+        private ConfigConvergenceChecker configConvergenceChecker = new ConfigConvergenceChecker();
+        private FlagSource flagSource = new InMemoryFlagSource();
 
         public DeployTester build() {
             Clock clock = Optional.ofNullable(this.clock).orElseGet(Clock::systemUTC);
@@ -285,9 +288,11 @@ public class DeployTester {
             ApplicationRepository applicationRepository = new ApplicationRepository.Builder()
                     .withTenantRepository(tenantRepository)
                     .withConfigserverConfig(configserverConfig)
-                    .withOrchestrator(Optional.ofNullable(orchestrator).orElseGet(OrchestratorMock::new))
+                    .withOrchestrator(new OrchestratorMock())
                     .withClock(clock)
                     .withProvisioner(provisioner)
+                    .withConfigConvergenceChecker(configConvergenceChecker)
+                    .withFlagSource(flagSource)
                     .build();
 
             return new DeployTester(clock, tenantRepository, applicationRepository);
@@ -336,10 +341,16 @@ public class DeployTester {
             return this;
         }
 
-        public Builder orchestrator(Orchestrator orchestrator) {
-            this.orchestrator = orchestrator;
+        public Builder configConvergenceChecker(ConfigConvergenceChecker configConvergenceChecker) {
+            this.configConvergenceChecker = configConvergenceChecker;
             return this;
         }
+
+        public Builder flagSource(FlagSource flagSource) {
+            this.flagSource = flagSource;
+            return this;
+        }
+
     }
 
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
index 33e18843738..143f9a8e80d 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
@@ -22,6 +22,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.config.server.MockConfigConvergenceChecker;
 import com.yahoo.vespa.config.server.application.ApplicationReindexing;
 import com.yahoo.vespa.config.server.http.InternalServerException;
 import com.yahoo.vespa.config.server.http.InvalidApplicationException;
@@ -29,6 +30,8 @@ import com.yahoo.vespa.config.server.http.UnknownVespaVersionException;
 import com.yahoo.vespa.config.server.http.v2.PrepareResult;
 import com.yahoo.vespa.config.server.model.TestModelFactory;
 import com.yahoo.vespa.config.server.session.PrepareParams;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.model.application.validation.change.VespaReindexAction;
 import com.yahoo.vespa.model.application.validation.change.VespaRestartAction;
 import org.junit.Rule;
@@ -432,7 +435,15 @@ public class HostedDeployTest {
                                                                           "reindex please", services, "music"),
                                                     new VespaRestartAction(ClusterSpec.Id.from("test"), "change", services)));
 
-        DeployTester tester = createTester(hosts, modelFactories, prodZone, clock);
+        DeployTester tester = new DeployTester.Builder()
+                .modelFactories(modelFactories)
+                .configserverConfig(createConfigserverConfig(prodZone))
+                .clock(clock)
+                .zone(prodZone)
+                .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true, false))
+                .configConvergenceChecker(new MockConfigConvergenceChecker(2))
+                .flagSource(new InMemoryFlagSource().withBooleanFlag(Flags.CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING.id(), true))
+                .build();
         PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.1.0");
 
         assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size());
@@ -480,7 +491,9 @@ public class HostedDeployTest {
                 .configserverConfig(createConfigserverConfig(prodZone))
                 .clock(clock)
                 .zone(prodZone)
-                .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true, false)).build();
+                .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true, false))
+                .configConvergenceChecker(new MockConfigConvergenceChecker(2))
+                .build();
     }
 
     private static class ConfigChangeActionsModelFactory extends TestModelFactory {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 04483e0191d..36930e27de4 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -556,8 +556,8 @@ public class ApplicationHandlerTest {
 
         { // Known service
             HttpResponse response = createResponse(new ServiceResponse(ServiceResponse.Status.ok,
-                                                                       3L,
-                                                                       3L,
+                                                                       3,
+                                                                       3,
                                                                        true),
                                                    hostAndPort,
                                                    uri);
@@ -601,9 +601,9 @@ public class ApplicationHandlerTest {
         {
             HttpServiceListResponse response =
                     new HttpServiceListResponse(new ServiceListResponse(Map.of(createServiceInfo(hostname, port), 3L),
-                                                                        requestUrl,
                                                                         3L,
-                                                                        3L));
+                                                                        3L),
+                                                requestUrl);
             assertResponse("{\n" +
                                    "  \"services\": [\n" +
                                    "    {\n" +
@@ -635,9 +635,9 @@ public class ApplicationHandlerTest {
 
             HttpServiceListResponse response =
                     new HttpServiceListResponse(new ServiceListResponse(serviceInfos,
-                                                                        requestUrl,
                                                                         4L,
-                                                                        3L));
+                                                                        3L),
+                                                requestUrl);
             assertResponse("{\n" +
                                    "  \"services\": [\n" +
                                    "    {\n" +