diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2021-07-06 13:05:53 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-07-06 13:05:53 +0200 |
commit | 75820e62e13fc08c7d9c009d9fd7da1a2a537a2b (patch) | |
tree | e3e9820096d30fc5227a38461f543130808d5743 /controller-server/src | |
parent | 9eecc4fe4fe9b9ccc6142b579d99fea1742065cf (diff) | |
parent | 58c24a53177df6bd02e56a5755de69a3d4b30539 (diff) |
Merge pull request #18535 from vespa-engine/jonmv/upgrade-dev-at-local-night
Upgrade dev at dep-local night
Diffstat (limited to 'controller-server/src')
11 files changed, 255 insertions, 25 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 25bc21a0076..28a6d32ce54 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -48,6 +48,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -230,6 +231,14 @@ public class JobController { return runs(id.application(), id.type()); } + /** Lists the start time of non-redeployment runs of the given job, in order of increasing age. */ + public List<Instant> jobStarts(JobId id) { + return runs(id).descendingMap().values().stream() + .filter(run -> ! run.isRedeployment()) + .map(Run::start) + .collect(toUnmodifiableList()); + } + /** Returns an immutable map of all known runs for the given application and job type. */ public NavigableMap<RunId, Run> runs(ApplicationId id, JobType type) { ImmutableSortedMap.Builder<RunId, Run> runs = ImmutableSortedMap.orderedBy(Comparator.comparing(RunId::number)); @@ -440,18 +449,23 @@ public class JobController { /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions) { - start(id, type, versions, JobProfile.of(type)); + start(id, type, versions, false); + } + + /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ + public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment) { + start(id, type, versions, isRedeployment, JobProfile.of(type)); } /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ - public void start(ApplicationId id, JobType type, Versions versions, JobProfile profile) { + public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, JobProfile profile) { locked(id, type, __ -> { Optional<Run> last = last(id, type); if (last.flatMap(run -> active(run.id())).isPresent()) throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!"); RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); - curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant(), profile)); + curator.writeLastRun(Run.initial(newId, versions, isRedeployment, controller.clock().instant(), profile)); metric.jobStarted(newId.job()); }); } @@ -477,6 +491,7 @@ public class JobController { ApplicationVersion.unknown, Optional.empty(), Optional.empty()), + false, JobProfile.development); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index d2481cd97ad..d93a0133c97 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -30,6 +30,7 @@ public class Run { private final RunId id; private final Map<Step, StepInfo> steps; private final Versions versions; + private final boolean isRedeployment; private final Instant start; private final Optional<Instant> end; private final RunStatus status; @@ -40,12 +41,13 @@ public class Run { private final Optional<X509Certificate> testerCertificate; // For deserialisation only -- do not use! - public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, Instant start, Optional<Instant> end, + public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.versions = versions; + this.isRedeployment = isRedeployment; this.start = start; this.end = end; this.status = status; @@ -56,10 +58,10 @@ public class Run { this.testerCertificate = testerCertificate; } - public static Run initial(RunId id, Versions versions, Instant now, JobProfile profile) { + public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile) { EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); - return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running, + return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), running, -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty()); } @@ -73,7 +75,7 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(Step.Status.of(status))); - return new Run(id, steps, versions, start, end, this.status == running ? status : this.status, + return new Run(id, steps, versions, isRedeployment, start, end, this.status == running ? status : this.status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } @@ -88,49 +90,49 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(startTime)); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run finished(Instant now) { requireActive(); - return new Run(id, steps, versions, start, Optional.of(now), status == running ? success : status, + return new Run(id, steps, versions, isRedeployment, start, Optional.of(now), status == running ? success : status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty()); } public Run aborted() { requireActive(); - return new Run(id, steps, versions, start, end, aborted, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, aborted, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run with(long lastTestRecord) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run with(Instant lastVespaLogTimestamp) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run noNodesDownSince(Instant noNodesDownSince) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate); } public Run withSummary(ConvergenceSummary convergenceSummary) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate); } public Run with(X509Certificate testerCertificate) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.of(testerCertificate)); } @@ -222,6 +224,11 @@ public class Run { return testerCertificate; } + /** Whether this is a automatic redeployment. */ + public boolean isRedeployment() { + return isRedeployment; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 91dfed500e3..56bf870c7fc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -42,6 +42,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(upgrader); maintainers.addAll(osUpgraders(controller, intervals.osUpgrader)); maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval)); + maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval)); maintainers.add(new DeploymentIssueReporter(controller, controller.serviceRegistry().deploymentIssues(), intervals.defaultInterval)); maintainers.add(new MetricsReporter(controller, metric)); maintainers.add(new OutstandingChangeDeployer(controller, intervals.outstandingChangeDeployer)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 9e3da506ca8..40191190eff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.yolean.Exceptions; import java.time.Duration; +import java.time.Instant; import java.util.Optional; import java.util.logging.Level; @@ -62,9 +63,8 @@ public class DeploymentExpirer extends ControllerMaintainer { .map(type -> new JobId(instance, type)); if (jobId.isEmpty()) return false; - return controller().jobController().last(jobId.get()) - .flatMap(Run::end) - .map(end -> end.plus(ttl.get()).isBefore(controller().clock().instant())) + return controller().jobController().jobStarts(jobId.get()).stream().findFirst() + .map(start -> start.plus(ttl.get()).isBefore(controller().clock().instant())) .orElse(false); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java new file mode 100644 index 00000000000..a69af024b96 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -0,0 +1,92 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.Versions; +import com.yahoo.yolean.Exceptions; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; + +/** + * Upgrades instances in manually deployed zones to the system version, at a convenient time. + * + * @author jonmv + */ +public class DeploymentUpgrader extends ControllerMaintainer { + + public DeploymentUpgrader(Controller controller, Duration interval) { + super(controller, interval); + } + + @Override + protected double maintain() { + AtomicInteger attempts = new AtomicInteger(); + AtomicInteger failures = new AtomicInteger(); + Versions target = new Versions(controller().readSystemVersion(), ApplicationVersion.unknown, Optional.empty(), Optional.empty()); + for (Application application : controller().applications().readable()) + for (Instance instance : application.instances().values()) + for (Deployment deployment : instance.deployments().values()) + try { + attempts.incrementAndGet(); + JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get()); + if ( ! deployment.zone().environment().isManuallyDeployed()) continue; + if ( ! deployment.version().isBefore(target.targetPlatform())) continue; + if ( controller().clock().instant().isBefore(controller().jobController().last(job).get().start().plus(Duration.ofDays(1)))) continue; + if ( ! isLikelyNightFor(job)) continue; + + log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); + controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true); + } catch (Exception e) { + failures.incrementAndGet(); + log.log(Level.WARNING, "Failed upgrading " + deployment + " of " + instance + + ": " + Exceptions.toMessageString(e) + ". Retrying in " + + interval()); + } + return asSuccessFactor(attempts.get(), failures.get()); + } + + private boolean isLikelyNightFor(JobId job) { + int hour = hourOf(controller().clock().instant()); + int[] runStarts = controller().jobController().jobStarts(job).stream() + .mapToInt(DeploymentUpgrader::hourOf) + .toArray(); + int localNight = mostLikelyWeeHour(runStarts); + return Math.abs(hour - localNight) <= 1; + } + + static int mostLikelyWeeHour(int[] starts) { + double weight = 1; + double[] buckets = new double[24]; + for (int start : starts) + buckets[start] += weight *= 0.8; // Weight more recent deployments higher. + + int best = -1; + double min = Double.MAX_VALUE; + for (int i = 12; i < 36; i++) { + double sum = 0; + for (int j = -12; j < 12; j++) + sum += buckets[(i + j) % 24] / (Math.abs(j) + 1); + + if (sum < min) { + min = sum; + best = i; + } + } + return (best + 2) % 24; + } + + private static int hourOf(Instant instant) { + return (int) (instant.toEpochMilli() / 3_600_000 % 24); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 87527085237..8ffa4823ead 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -83,6 +83,7 @@ class RunSerializer { private static final String endField = "end"; private static final String statusField = "status"; private static final String versionsField = "versions"; + private static final String isRedeploymentField = "isRedeployment"; private static final String platformVersionField = "platform"; private static final String repositoryField = "repository"; private static final String branchField = "branch"; @@ -131,6 +132,7 @@ class RunSerializer { runObject.field(numberField).asLong()), steps, versionsFromSlime(runObject.field(versionsField)), + runObject.field(isRedeploymentField).asBool(), SlimeUtils.instant(runObject.field(startField)), SlimeUtils.optionalInstant(runObject.field(endField)), runStatusOf(runObject.field(statusField).asString()), @@ -177,7 +179,7 @@ class RunSerializer { compileVersion, buildTime, sourceUrl, commit); } - // Don't change this — introduce a separate array instead. + // Don't change this — introduce a separate array instead. private Optional<ConvergenceSummary> convergenceSummaryFrom(Inspector summaryArray) { if ( ! summaryArray.valid()) return Optional.empty(); @@ -215,6 +217,7 @@ class RunSerializer { private void toSlime(Run run, Cursor runObject) { runObject.setString(applicationField, run.id().application().serializedForm()); runObject.setString(jobTypeField, run.id().type().jobName()); + runObject.setBool(isRedeploymentField, run.isRedeployment()); runObject.setLong(numberField, run.id().number()); runObject.setLong(startField, run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 054c5f21b0c..21b0fb12a5c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1387,9 +1387,10 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); response.setString("version", deployment.version().toFullString()); response.setString("revision", deployment.applicationVersion().id()); - response.setLong("deployTimeEpochMs", deployment.at().toEpochMilli()); + Instant lastDeploymentStart = lastDeploymentStart(deploymentId.applicationId(), deployment); + response.setLong("deployTimeEpochMs", lastDeploymentStart.toEpochMilli()); controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId()) - .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", deployment.at().plus(deploymentTimeToLive).toEpochMilli())); + .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", lastDeploymentStart.plus(deploymentTimeToLive).toEpochMilli())); application.projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); sourceRevisionToSlime(deployment.applicationVersion().source(), response); @@ -1441,6 +1442,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { metrics.instant().ifPresent(instant -> metricsObject.setLong("lastUpdated", instant.toEpochMilli())); } + private Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) { + return controller.jobController().jobStarts(new JobId(instanceId, JobType.from(controller.system(), deployment.zone()).get())) + .stream().findFirst().orElse(deployment.at()); + } + private void toSlime(ApplicationVersion applicationVersion, Cursor object) { if ( ! applicationVersion.isUnknown()) { object.setLong("buildNumber", applicationVersion.buildNumber().getAsLong()); @@ -2193,9 +2199,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private void tenantMetaDataToSlime(Tenant tenant, List<Application> applications, Cursor object) { Optional<Instant> lastDev = applications.stream() .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> instance.deployments().values().stream()) - .filter(deployment -> deployment.zone().environment() == Environment.dev) - .map(Deployment::at) + .flatMap(instance -> instance.deployments().values().stream() + .filter(deployment -> deployment.zone().environment() == Environment.dev) + .map(deployment -> lastDeploymentStart(instance.id(), deployment))) .max(Comparator.naturalOrder()) .or(() -> applications.stream() .flatMap(application -> application.instances().values().stream()) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index ffc82f90ad4..31f8aaf9e2d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -70,7 +70,9 @@ public class DeploymentExpirerTest { assertEquals(1, permanentDeployments(prodApp.instance())); // Dev application expires when enough time has passed since most recent attempt + // Redeployments done by DeploymentUpgrader do not affect this tester.clock().advance(Duration.ofDays(12).plus(Duration.ofSeconds(1))); + tester.jobs().start(devApp.instanceId(), JobType.devUsEast1, lastRun.versions(), true); expirer.maintain(); assertEquals(0, permanentDeployments(devApp.instance())); assertEquals(1, permanentDeployments(prodApp.instance())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java new file mode 100644 index 00000000000..552a38e6678 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java @@ -0,0 +1,101 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentUpgrader.mostLikelyWeeHour; +import static java.time.temporal.ChronoUnit.MILLIS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author jonmv + */ +public class DeploymentUpgraderTest { + + private final DeploymentTester tester = new DeploymentTester(); + + @Test + public void testDeploymentUpgrading() { + ZoneId devZone = ZoneId.from(Environment.dev, RegionName.from("us-east-1")); + DeploymentUpgrader upgrader = new DeploymentUpgrader(tester.controller(), Duration.ofDays(1)); + var devApp = tester.newDeploymentContext("tenant1", "app1", "default"); + var prodApp = tester.newDeploymentContext("tenant2", "app2", "default"); + + ApplicationPackage appPackage = new ApplicationPackageBuilder().region("us-west-1").build(); + Version systemVersion = tester.controller().readSystemVersion(); + Instant start = tester.clock().instant().truncatedTo(MILLIS); + + devApp.runJob(devUsEast1, appPackage); + prodApp.submit(appPackage).deploy(); + assertEquals(systemVersion, tester.jobs().last(devApp.instanceId(), devUsEast1).get().versions().targetPlatform()); + assertEquals(systemVersion, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().versions().targetPlatform()); + + // Not upgraded initially + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // Not upgraded immediately after system upgrades + tester.controllerTester().upgradeSystem(new Version(7, 8, 9)); + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // 13 hours pass, but not upgraded before a day has passed since last deployment + tester.clock().advance(Duration.ofHours(13)); + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // 34 hours pass, but not upgraded since it's not likely in the middle of the night + tester.clock().advance(Duration.ofHours(21)); + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // 37 hours pass, and the dev deployment, only, is upgraded + tester.clock().advance(Duration.ofHours(3)); + upgrader.maintain(); + assertEquals(tester.clock().instant().truncatedTo(MILLIS), tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertTrue(tester.jobs().last(devApp.instanceId(), devUsEast1).get().isRedeployment()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + devApp.runJob(devUsEast1); + + // After the upgrade, the dev app is mostly (re)deployed to at night, but this doesn't affect what is likely the night. + tester.controllerTester().upgradeSystem(new Version(7, 9, 11)); + tester.clock().advance(Duration.ofHours(48)); + upgrader.maintain(); + assertEquals(tester.clock().instant().truncatedTo(MILLIS), tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + } + + @Test + public void testNight() { + assertEquals(16, mostLikelyWeeHour(new int[]{ 0, 1, 2, 3, 4, 5, 6 })); + assertEquals(14, mostLikelyWeeHour(new int[]{ 22, 23, 0, 1, 2, 3, 4 })); + assertEquals(18, mostLikelyWeeHour(new int[]{ 6, 5, 4, 3, 2, 1, 0 })); + assertEquals(20, mostLikelyWeeHour(new int[]{ 0, 12, 0, 12, 0, 12, 0, 12, 0, 12, 0, 11 })); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 6e12373640d..03a050db74e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -149,7 +149,7 @@ public class RunSerializerTest { assertEquals(run.versions(), phoenix.versions()); assertEquals(run.steps(), phoenix.steps()); - Run initial = Run.initial(id, run.versions(), run.start(), JobProfile.production); + Run initial = Run.initial(id, run.versions(), run.isRedeployment(), run.start(), JobProfile.production); assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index e906df94023..668baa50cc1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -43,6 +43,9 @@ "name": "DeploymentMetricsMaintainer" }, { + "name": "DeploymentUpgrader" + }, + { "name": "EndpointCertificateMaintainer" }, { |