From 98961445490284cea19abbb21339c71b8df7d2cf Mon Sep 17 00:00:00 2001 From: Andreas Eriksen Date: Tue, 2 Aug 2022 09:40:17 +0200 Subject: trigger one prod job per run --- .../vespa/hosted/controller/maintenance/ControllerMaintenance.java | 2 +- .../controller/maintenance/EndpointCertificateMaintainer.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'controller-server/src/main') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 9793cded918..a259ed2fdef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -156,7 +156,7 @@ public class ControllerMaintenance extends AbstractComponent { this.containerImageExpirer = duration(12, HOURS); this.hostInfoUpdater = duration(12, HOURS); this.reindexingTriggerer = duration(1, HOURS); - this.endpointCertificateMaintainer = duration(12, HOURS); + this.endpointCertificateMaintainer = duration(1, HOURS); this.trafficFractionUpdater = duration(5, MINUTES); this.archiveUriUpdater = duration(5, MINUTES); this.archiveAccessMaintainer = duration(10, MINUTES); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index f3256237284..0e726241d77 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -93,17 +94,18 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { } /** - * If it's been four days since the cert has been refreshed, re-trigger all prod deployment jobs. + * If it's been four days since the cert has been refreshed, re-trigger prod deployment jobs (one at a time). */ private void deployRefreshedCertificates() { var now = clock.instant(); + var jobsTriggered = new AtomicInteger(0); curator.readAllEndpointCertificateMetadata().forEach((applicationId, endpointCertificateMetadata) -> endpointCertificateMetadata.lastRefreshed().ifPresent(lastRefreshTime -> { Instant refreshTime = Instant.ofEpochSecond(lastRefreshTime); if (now.isAfter(refreshTime.plus(4, ChronoUnit.DAYS))) { controller().applications().getInstance(applicationId) .ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> { - if (deployment.at().isBefore(refreshTime)) { + if (deployment.at().isBefore(refreshTime) && jobsTriggered.compareAndSet(0, 1)) { JobType job = JobType.deploymentTo(zone); deploymentTrigger.reTrigger(applicationId, job, "re-triggered by EndpointCertificateMaintainer"); log.info("Re-triggering deployment job " + job.jobName() + " for instance " + -- cgit v1.2.3 From 524ec3aca0fafd8a5ad31ecc99381bb895e133e3 Mon Sep 17 00:00:00 2001 From: Andreas Eriksen Date: Tue, 2 Aug 2022 12:13:09 +0200 Subject: trigger oldest unrefreshed job on each run --- .../maintenance/EndpointCertificateMaintainer.java | 28 +++++++++++++++------- .../EndpointCertificateMaintainerTest.java | 28 ++++++++++++++++++++++ 2 files changed, 47 insertions(+), 9 deletions(-) (limited to 'controller-server/src/main') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 0e726241d77..2e2680cd34a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateDetails; @@ -15,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCe import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequestMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -23,12 +23,13 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; -import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -49,6 +50,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { private final CuratorDb curator; private final SecretStore secretStore; private final EndpointCertificateProvider endpointCertificateProvider; + final Comparator oldestFirst = Comparator.comparing(e -> e.deployment.at()); @Inject public EndpointCertificateMaintainer(Controller controller, Duration interval) { @@ -93,27 +95,35 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { })); } + record EligibleJob(Deployment deployment, ApplicationId applicationId, JobType job) {} /** * If it's been four days since the cert has been refreshed, re-trigger prod deployment jobs (one at a time). */ private void deployRefreshedCertificates() { var now = clock.instant(); - var jobsTriggered = new AtomicInteger(0); + var eligibleJobs = new ArrayList(); + curator.readAllEndpointCertificateMetadata().forEach((applicationId, endpointCertificateMetadata) -> endpointCertificateMetadata.lastRefreshed().ifPresent(lastRefreshTime -> { Instant refreshTime = Instant.ofEpochSecond(lastRefreshTime); if (now.isAfter(refreshTime.plus(4, ChronoUnit.DAYS))) { controller().applications().getInstance(applicationId) .ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> { - if (deployment.at().isBefore(refreshTime) && jobsTriggered.compareAndSet(0, 1)) { + if (deployment.at().isBefore(refreshTime)) { JobType job = JobType.deploymentTo(zone); - deploymentTrigger.reTrigger(applicationId, job, "re-triggered by EndpointCertificateMaintainer"); - log.info("Re-triggering deployment job " + job.jobName() + " for instance " + - applicationId.serializedForm() + " to roll out refreshed endpoint certificate"); + eligibleJobs.add(new EligibleJob(deployment, applicationId, job)); } })); } })); + + eligibleJobs.stream() + .min(oldestFirst) + .ifPresent(e -> { + deploymentTrigger.reTrigger(e.applicationId, e.job, "re-triggered by EndpointCertificateMaintainer"); + log.info("Re-triggering deployment job " + e.job.jobName() + " for instance " + + e.applicationId.serializedForm() + " to roll out refreshed endpoint certificate"); + }); } private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) { @@ -158,8 +168,8 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { List endpointCertificateMetadata = endpointCertificateProvider.listCertificates(); Map storedEndpointCertificateMetadata = curator.readAllEndpointCertificateMetadata(); - List leafRequestIds = storedEndpointCertificateMetadata.values().stream().flatMap(m -> m.leafRequestId().stream()).collect(Collectors.toList()); - List rootRequestIds = storedEndpointCertificateMetadata.values().stream().map(EndpointCertificateMetadata::rootRequestId).collect(Collectors.toList()); + List leafRequestIds = storedEndpointCertificateMetadata.values().stream().flatMap(m -> m.leafRequestId().stream()).toList(); + List rootRequestIds = storedEndpointCertificateMetadata.values().stream().map(EndpointCertificateMetadata::rootRequestId).toList(); for (var providerCertificateMetadata : endpointCertificateMetadata) { if (!rootRequestIds.contains(providerCertificateMetadata.requestId()) && !leafRequestIds.contains(providerCertificateMetadata.requestId())) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index c103894b1a3..47a1b44d196 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -1,19 +1,32 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.Test; import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Optional; +import java.util.OptionalDouble; +import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; @@ -120,6 +133,21 @@ public class EndpointCertificateMaintainerTest { deploymentContext.assertRunning(productionUsWest1); } + @Test + void testEligibleSorting() { + EndpointCertificateMaintainer.EligibleJob oldestDeployment = makeDeploymentAtAge(5); + assertEquals( + oldestDeployment, + Stream.of(makeDeploymentAtAge(2), oldestDeployment, makeDeploymentAtAge(4)).min(maintainer.oldestFirst).get()); + } + + @NotNull + private EndpointCertificateMaintainer.EligibleJob makeDeploymentAtAge(int ageInDays) { + var deployment = new Deployment(ZoneId.defaultId(), RevisionId.forProduction(1), Version.emptyVersion, + Instant.now().minus(ageInDays, ChronoUnit.DAYS), DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty()); + return new EndpointCertificateMaintainer.EligibleJob(deployment, ApplicationId.defaultId(), JobType.prod("somewhere")); + } + @Test void unmaintained_cert_is_deleted() { EndpointCertificateMock endpointCertificateProvider = (EndpointCertificateMock) tester.controller().serviceRegistry().endpointCertificateProvider(); -- cgit v1.2.3