diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2022-01-17 14:39:49 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-01-17 14:39:49 +0100 |
commit | ce7cd3cb0479f84851d0ac6500aab2271eca424e (patch) | |
tree | 176c9a9258a347025dedc0c6843c8151ab47980c /configserver | |
parent | e7f13b646891ddbfad4dfbc5725bfbcbc34d1132 (diff) | |
parent | 1c1d2d64438e8f034c455c154bc45cd983e872a2 (diff) |
Merge pull request #20832 from vespa-engine/hmusum/wait-for-config-generation-before-restarting
Hmusum/wait for config generation before restarting [run-systemtest]
Diffstat (limited to 'configserver')
13 files changed, 197 insertions, 44 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 63ab00275bf..2ee3e6c0343 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -88,7 +88,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; -import java.net.URI; import java.nio.file.Files; import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; @@ -111,8 +110,8 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; -import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk; import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT; @@ -147,6 +146,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final Metric metric; private final SecretStoreValidator secretStoreValidator; private final ClusterReindexingStatusClient clusterReindexingStatusClient; + private final FlagSource flagSource; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -203,8 +203,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.metric = Objects.requireNonNull(metric); this.secretStoreValidator = Objects.requireNonNull(secretStoreValidator); this.clusterReindexingStatusClient = clusterReindexingStatusClient; + this.flagSource = flagSource; } + // Should be used by tests only (first constructor in this class makes sure we use injectable components where possible) public static class Builder { private TenantRepository tenantRepository; private Optional<Provisioner> hostProvisioner; @@ -217,6 +219,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private Metric metric = new NullMetric(); private SecretStoreValidator secretStoreValidator = new SecretStoreValidator(new SecretStoreProvider().get()); private FlagSource flagSource = new InMemoryFlagSource(); + private ConfigConvergenceChecker configConvergenceChecker = new ConfigConvergenceChecker(); public Builder withTenantRepository(TenantRepository tenantRepository) { this.tenantRepository = tenantRepository; @@ -280,11 +283,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return this; } + public Builder withConfigConvergenceChecker(ConfigConvergenceChecker configConvergenceChecker) { + this.configConvergenceChecker = configConvergenceChecker; + return this; + } + public ApplicationRepository build() { return new ApplicationRepository(tenantRepository, hostProvisioner, InfraDeployerProvider.empty().getInfraDeployer(), - new ConfigConvergenceChecker(), + configConvergenceChecker, httpProxy, configserverConfig, orchestrator, @@ -747,10 +755,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public ServiceListResponse servicesToCheckForConfigConvergence(ApplicationId applicationId, - URI uri, Duration timeoutPerService, Optional<Version> vespaVersion) { - return convergeChecker.getServiceConfigGenerations(getApplication(applicationId, vespaVersion), uri, timeoutPerService); + return convergeChecker.getConfigGenerationsForAllServices(getApplication(applicationId, vespaVersion), timeoutPerService); } public ConfigConvergenceChecker configConvergenceChecker() { return convergeChecker; } @@ -1016,6 +1023,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return applicationId.orElse(null); } + public FlagSource flagSource() { return flagSource; } + private Session validateThatLocalSessionIsNotActive(Tenant tenant, long sessionId) { Session session = getLocalSession(tenant, sessionId); if (Session.Status.ACTIVATE.equals(session.getStatus())) { @@ -1042,6 +1051,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return getTenant(appId).getSessionRepository().getActiveApplicationSet(appId); } + public Application getActiveApplication(ApplicationId applicationId) { + return getActiveApplicationSet(applicationId) + .map(a -> a.getForVersionOrLatest(Optional.empty(), clock.instant())) + .orElseThrow(() -> new RuntimeException("Found no active application for " + applicationId)); + } + private File decompressApplication(InputStream in, String contentType, File tempDir) { try (CompressedApplicationInputStream application = CompressedApplicationInputStream.createFromCompressedStream(in, contentType)) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index ad14cf4aab6..d8dc26b0b3a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -39,8 +39,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -71,7 +72,7 @@ public class ConfigConvergenceChecker extends AbstractComponent { ); - private final Executor responseHandlerExecutor = + private final ExecutorService responseHandlerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config-convergence-checker-response-handler-")); private final ObjectMapper jsonMapper = new ObjectMapper(); @@ -90,10 +91,10 @@ public class ConfigConvergenceChecker extends AbstractComponent { } /** Check all services in given application. Returns the minimum current generation of all services */ - public ServiceListResponse getServiceConfigGenerations(Application application, URI uri, Duration timeoutPerService) { + public ServiceListResponse getConfigGenerationsForAllServices(Application application, Duration timeoutPerService) { Map<ServiceInfo, Long> currentGenerations = getServiceConfigGenerations(application, timeoutPerService); long currentGeneration = currentGenerations.values().stream().mapToLong(Long::longValue).min().orElse(-1); - return new ServiceListResponse(currentGenerations, uri, application.getApplicationGeneration(), currentGeneration); + return new ServiceListResponse(currentGenerations, application.getApplicationGeneration(), currentGeneration); } /** Check service identified by host and port in given application */ @@ -196,6 +197,16 @@ public class ConfigConvergenceChecker extends AbstractComponent { .findFirst(); } + @Override + public void deconstruct() { + responseHandlerExecutor.shutdown(); + try { + responseHandlerExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + log.log(Level.WARNING, "Unable to shutdown executor", e); + } + } + private static long generationFromContainerState(JsonNode state) { return state.get("config").get("generation").asLong(-1); } @@ -256,23 +267,23 @@ public class ConfigConvergenceChecker extends AbstractComponent { public final boolean converged; public final Optional<String> errorMessage; - public ServiceResponse(Status status, Long wantedGeneration) { - this(status, wantedGeneration, 0L); + public ServiceResponse(Status status, long wantedGeneration) { + this(status, wantedGeneration, 0); } - public ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration) { + public ServiceResponse(Status status, long wantedGeneration, long currentGeneration) { this(status, wantedGeneration, currentGeneration, false); } - public ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration, boolean converged) { + public ServiceResponse(Status status, long wantedGeneration, long currentGeneration, boolean converged) { this(status, wantedGeneration, currentGeneration, converged, Optional.empty()); } - public ServiceResponse(Status status, Long wantedGeneration, String errorMessage) { - this(status, wantedGeneration, 0L, false, Optional.ofNullable(errorMessage)); + public ServiceResponse(Status status, long wantedGeneration, String errorMessage) { + this(status, wantedGeneration, 0, false, Optional.ofNullable(errorMessage)); } - private ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration, boolean converged, Optional<String> errorMessage) { + private ServiceResponse(Status status, long wantedGeneration, long currentGeneration, boolean converged, Optional<String> errorMessage) { this.status = status; this.wantedGeneration = wantedGeneration; this.currentGeneration = currentGeneration; @@ -285,15 +296,15 @@ public class ConfigConvergenceChecker extends AbstractComponent { public static class ServiceListResponse { public final List<Service> services = new ArrayList<>(); - public final URI uri; public final long wantedGeneration; public final long currentGeneration; + public final boolean converged; - public ServiceListResponse(Map<ServiceInfo, Long> services, URI uri, long wantedGeneration, long currentGeneration) { + public ServiceListResponse(Map<ServiceInfo, Long> services, long wantedGeneration, long currentGeneration) { services.forEach((key, value) -> this.services.add(new Service(key, value))); - this.uri = uri; this.wantedGeneration = wantedGeneration; this.currentGeneration = currentGeneration; + this.converged = currentGeneration >= wantedGeneration; } public List<Service> services() { return services; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java index f0711e5c238..88cddb93d9d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java @@ -5,7 +5,13 @@ package com.yahoo.vespa.config.server.application; * @author Ulf Lilleengen */ public class ConfigNotConvergedException extends RuntimeException { + + public ConfigNotConvergedException(Throwable t) { + super(t); + } + public ConfigNotConvergedException(String message) { super(message); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 4b7e39f48a4..49dade6f0cd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.deploy; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ServiceInfo; @@ -19,12 +20,17 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.ApplicationRepository.Activation; import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; +import com.yahoo.vespa.config.server.application.ConfigNotConvergedException; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.ReindexActions; import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.Flags; import java.time.Clock; import java.time.Duration; @@ -35,6 +41,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; + /** * The process of deploying an application. * Deployments are created by an {@link ApplicationRepository}. @@ -149,6 +157,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { RestartActions restartActions = configChangeActions.getRestartActions().useForInternalRestart(internalRedeploy); if ( ! restartActions.isEmpty()) { + + waitForConfigToConverge(applicationId); + Set<String> hostnames = restartActions.getEntries().stream() .flatMap(entry -> entry.getServices().stream()) .map(ServiceInfo::getHostName) @@ -165,6 +176,39 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } } + private void waitForConfigToConverge(ApplicationId applicationId) { + BooleanFlag verify = Flags.CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING.bindTo(applicationRepository.flagSource()); + if ( ! verify.value()) return; + + // Timeout per service when getting config generations + Duration timeout = Duration.ofSeconds(10); + while (true) { + try { + params.get().getTimeoutBudget().assertNotTimedOut( + () -> "Timeout exceeded while waiting for config convergence for " + applicationId); + } catch (UncheckedTimeoutException e) { + throw new ConfigNotConvergedException(e); + } + + Application app = applicationRepository.getActiveApplication(applicationId); + + // TODO: Don't wait for config convergence if restartOnDeploy is true for one o more container clusters + // (ideally wait for config convergence for all other services) + + log.info(session.logPre() + "Wait for services to converge on new generation before restarting"); + ConfigConvergenceChecker convergenceChecker = applicationRepository.configConvergenceChecker(); + ServiceListResponse response = convergenceChecker.getConfigGenerationsForAllServices(app, timeout); + if (response.converged) { + log.info(session.logPre() + "Services converged on new generation " + response.currentGeneration); + return; + } else { + log.info(session.logPre() + "Services not converged on new generation, wanted generation: " + response.wantedGeneration + + ", current generation: " + response.currentGeneration + ", will retry"); + try { Thread.sleep(10_000); } catch (InterruptedException e) { /* ignore */ } + } + } + } + private void storeReindexing(ApplicationId applicationId, long requiredSession) { applicationRepository.modifyReindexing(applicationId, reindexing -> { if (configChangeActions != null) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java index 96e1767b462..c414d8dfa9a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -2,13 +2,13 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.container.jdisc.HttpResponse; -import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import java.io.IOException; import java.io.OutputStream; +import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; @@ -50,7 +50,8 @@ public class HttpErrorResponse extends HttpResponse { UNKNOWN_VESPA_VERSION, PARENT_HOST_NOT_READY, CERTIFICATE_NOT_READY, - LOAD_BALANCER_NOT_READY + LOAD_BALANCER_NOT_READY, + CONFIG_NOT_CONVERGED } public static HttpErrorResponse notFoundError(String msg) { @@ -101,6 +102,10 @@ public class HttpErrorResponse extends HttpResponse { return new HttpErrorResponse(CONFLICT, ErrorCode.CERTIFICATE_NOT_READY.name(), msg); } + public static HttpErrorResponse configNotConverged(String msg) { + return new HttpErrorResponse(CONFLICT, ErrorCode.CONFIG_NOT_CONVERGED.name(), msg); + } + public static HttpErrorResponse loadBalancerNotReady(String msg) { return new HttpErrorResponse(CONFLICT, ErrorCode.LOAD_BALANCER_NOT_READY.name(), msg); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index 2dad2c060cc..9f4f8b5f9ce 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.vespa.config.server.application.ConfigNotConvergedException; import com.yahoo.yolean.Exceptions; import java.io.PrintWriter; @@ -68,6 +69,8 @@ public class HttpHandler extends LoggingRequestHandler { return HttpErrorResponse.parentHostNotReady(getMessage(e, request)); } catch (CertificateNotReadyException e) { return HttpErrorResponse.certificateNotReady(getMessage(e, request)); + } catch (ConfigNotConvergedException e) { + return HttpErrorResponse.configNotConverged(getMessage(e, request)); } catch (LoadBalancerServiceException e) { return HttpErrorResponse.loadBalancerNotReady(getMessage(e, request)); } catch (Exception e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 0131517818d..df4df234ed0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -117,10 +117,9 @@ public class ApplicationHandler extends HttpHandler { private HttpResponse listServiceConverge(ApplicationId applicationId, HttpRequest request) { ServiceListResponse response = applicationRepository.servicesToCheckForConfigConvergence(applicationId, - request.getUri(), getTimeoutFromRequest(request), getVespaVersionFromRequest(request)); - return new HttpServiceListResponse(response); + return new HttpServiceListResponse(response, request.getUri()); } private HttpResponse checkServiceConverge(ApplicationId applicationId, String hostAndPort, HttpRequest request) { @@ -370,7 +369,7 @@ public class ApplicationHandler extends HttpHandler { static class HttpServiceListResponse extends JSONResponse { // Pre-condition: servicesToCheck has a state port - public HttpServiceListResponse(ConfigConvergenceChecker.ServiceListResponse response) { + public HttpServiceListResponse(ConfigConvergenceChecker.ServiceListResponse response, URI uri) { super(200); Cursor serviceArray = object.setArray("services"); response.services().forEach((service) -> { @@ -381,13 +380,13 @@ public class ApplicationHandler extends HttpHandler { serviceObject.setString("host", hostName); serviceObject.setLong("port", statePort); serviceObject.setString("type", serviceInfo.getServiceType()); - serviceObject.setString("url", response.uri.toString() + "/" + hostName + ":" + statePort); + serviceObject.setString("url", uri.toString() + "/" + hostName + ":" + statePort); serviceObject.setLong("currentGeneration", service.currentGeneration); }); - object.setString("url", response.uri.toString()); + object.setString("url", uri.toString()); object.setLong("currentGeneration", response.currentGeneration); object.setLong("wantedGeneration", response.wantedGeneration); - object.setBool("converged", response.currentGeneration >= response.wantedGeneration); + object.setBool("converged", response.converged); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index c2af7f1ee1f..284a0aa0a62 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -103,6 +103,7 @@ public class ApplicationRepositoryTest { private OrchestratorMock orchestrator; private TimeoutBudget timeoutBudget; private Curator curator; + private ConfigserverConfig configserverConfig; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -114,7 +115,7 @@ public class ApplicationRepositoryTest { @Before public void setup() throws IOException { curator = new MockCurator(); - ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + configserverConfig = new ConfigserverConfig.Builder() .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) @@ -170,6 +171,16 @@ public class ApplicationRepositoryTest { @Test public void prepareAndActivateWithRestart() { + applicationRepository = new ApplicationRepository.Builder() + .withTenantRepository(tenantRepository) + .withProvisioner(provisioner) + .withConfigserverConfig(configserverConfig) + .withOrchestrator(orchestrator) + .withLogRetriever(new MockLogRetriever()) + .withClock(clock) + .withConfigConvergenceChecker(new MockConfigConvergenceChecker(2)) + .build(); + prepareAndActivate(testAppJdiscOnly); PrepareResult result = prepareAndActivate(testAppJdiscOnlyRestart); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java new file mode 100644 index 00000000000..b21d89fa626 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java @@ -0,0 +1,33 @@ +package com.yahoo.vespa.config.server; + +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; + +import java.time.Duration; +import java.util.Map; + +public class MockConfigConvergenceChecker extends ConfigConvergenceChecker { + + private final long wantedGeneration; + + public MockConfigConvergenceChecker(long wantedGeneration) { + this.wantedGeneration = wantedGeneration; + } + + @Override + public Map<ServiceInfo, Long> getServiceConfigGenerations(Application application, Duration timeoutPerService) { + return Map.of(); + } + + @Override + public ServiceListResponse getConfigGenerationsForAllServices(Application application, Duration timeoutPerService) { + return new ServiceListResponse(Map.of(), wantedGeneration, wantedGeneration); + } + + @Override + public ServiceResponse getServiceConfigGeneration(Application application, String hostAndPortToCheck, Duration timeout) { + return new ServiceResponse(ServiceResponse.Status.ok, wantedGeneration); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java index 6afb9ef086d..148cd8ac24f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java @@ -28,6 +28,7 @@ import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -88,12 +89,12 @@ public class ConfigConvergenceCheckerTest { @Test public void service_list_convergence() { { - URI requestUrl = testServer().resolve("/serviceconverge"); wireMock.stubFor(get(urlEqualTo("/state/v1/config")).willReturn(okJson("{\"config\":{\"generation\":3}}"))); - ServiceListResponse response = checker.getServiceConfigGenerations(application, requestUrl, clientTimeout); + ServiceListResponse response = checker.getConfigGenerationsForAllServices(application, clientTimeout); assertEquals(3, response.wantedGeneration); assertEquals(3, response.currentGeneration); + assertTrue(response.converged); List<ServiceListResponse.Service> services = response.services; assertEquals(1, services.size()); assertService(this.service, services.get(0), 3); @@ -113,9 +114,10 @@ public class ConfigConvergenceCheckerTest { URI requestUrl = testServer().resolve("/serviceconverge"); - ServiceListResponse response = checker.getServiceConfigGenerations(application, requestUrl, clientTimeout); + ServiceListResponse response = checker.getConfigGenerationsForAllServices(application, clientTimeout); assertEquals(4, response.wantedGeneration); assertEquals(3, response.currentGeneration); + assertFalse(response.converged); List<ServiceListResponse.Service> services = response.services; assertEquals(2, services.size()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index f16b1102f2a..7f223fbce6f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -24,6 +24,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.http.v2.PrepareResult; @@ -37,9 +38,10 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; -import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; import java.nio.file.Files; @@ -253,7 +255,8 @@ public class DeployTester { private Curator curator = new MockCurator(); private Metrics metrics; private List<ModelFactory> modelFactories; - private Orchestrator orchestrator; + private ConfigConvergenceChecker configConvergenceChecker = new ConfigConvergenceChecker(); + private FlagSource flagSource = new InMemoryFlagSource(); public DeployTester build() { Clock clock = Optional.ofNullable(this.clock).orElseGet(Clock::systemUTC); @@ -285,9 +288,11 @@ public class DeployTester { ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withConfigserverConfig(configserverConfig) - .withOrchestrator(Optional.ofNullable(orchestrator).orElseGet(OrchestratorMock::new)) + .withOrchestrator(new OrchestratorMock()) .withClock(clock) .withProvisioner(provisioner) + .withConfigConvergenceChecker(configConvergenceChecker) + .withFlagSource(flagSource) .build(); return new DeployTester(clock, tenantRepository, applicationRepository); @@ -336,10 +341,16 @@ public class DeployTester { return this; } - public Builder orchestrator(Orchestrator orchestrator) { - this.orchestrator = orchestrator; + public Builder configConvergenceChecker(ConfigConvergenceChecker configConvergenceChecker) { + this.configConvergenceChecker = configConvergenceChecker; return this; } + + public Builder flagSource(FlagSource flagSource) { + this.flagSource = flagSource; + return this; + } + } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 33e18843738..143f9a8e80d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -22,6 +22,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.config.server.MockConfigConvergenceChecker; import com.yahoo.vespa.config.server.application.ApplicationReindexing; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.InvalidApplicationException; @@ -29,6 +30,8 @@ import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.application.validation.change.VespaReindexAction; import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import org.junit.Rule; @@ -432,7 +435,15 @@ public class HostedDeployTest { "reindex please", services, "music"), new VespaRestartAction(ClusterSpec.Id.from("test"), "change", services))); - DeployTester tester = createTester(hosts, modelFactories, prodZone, clock); + DeployTester tester = new DeployTester.Builder() + .modelFactories(modelFactories) + .configserverConfig(createConfigserverConfig(prodZone)) + .clock(clock) + .zone(prodZone) + .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true, false)) + .configConvergenceChecker(new MockConfigConvergenceChecker(2)) + .flagSource(new InMemoryFlagSource().withBooleanFlag(Flags.CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING.id(), true)) + .build(); PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.1.0"); assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); @@ -480,7 +491,9 @@ public class HostedDeployTest { .configserverConfig(createConfigserverConfig(prodZone)) .clock(clock) .zone(prodZone) - .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true, false)).build(); + .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true, false)) + .configConvergenceChecker(new MockConfigConvergenceChecker(2)) + .build(); } private static class ConfigChangeActionsModelFactory extends TestModelFactory { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 04483e0191d..36930e27de4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -556,8 +556,8 @@ public class ApplicationHandlerTest { { // Known service HttpResponse response = createResponse(new ServiceResponse(ServiceResponse.Status.ok, - 3L, - 3L, + 3, + 3, true), hostAndPort, uri); @@ -601,9 +601,9 @@ public class ApplicationHandlerTest { { HttpServiceListResponse response = new HttpServiceListResponse(new ServiceListResponse(Map.of(createServiceInfo(hostname, port), 3L), - requestUrl, 3L, - 3L)); + 3L), + requestUrl); assertResponse("{\n" + " \"services\": [\n" + " {\n" + @@ -635,9 +635,9 @@ public class ApplicationHandlerTest { HttpServiceListResponse response = new HttpServiceListResponse(new ServiceListResponse(serviceInfos, - requestUrl, 4L, - 3L)); + 3L), + requestUrl); assertResponse("{\n" + " \"services\": [\n" + " {\n" + |