From 52ce734cbf85f0e71e8de7ac6e61979d5ff0bcff Mon Sep 17 00:00:00 2001 From: bjormel Date: Wed, 29 Mar 2023 16:06:43 +0000 Subject: maintainer success factor baseline deviation --- .../maintenance/ApplicationOwnershipConfirmer.java | 6 +++--- .../controller/maintenance/ArchiveUriUpdater.java | 2 +- .../controller/maintenance/ArtifactExpirer.java | 4 ++-- .../controller/maintenance/BcpGroupUpdater.java | 14 +++++++------- .../maintenance/BillingDatabaseMaintainer.java | 2 +- .../maintenance/CloudDatabaseMaintainer.java | 4 ++-- .../maintenance/ContactInformationMaintainer.java | 2 +- .../controller/maintenance/ControllerMaintainer.java | 18 +++++++++++++----- .../maintenance/ControllerMaintenance.java | 14 +++++++++++++- .../controller/maintenance/CostReportMaintainer.java | 2 +- .../controller/maintenance/DeploymentExpirer.java | 4 +--- .../maintenance/DeploymentInfoMaintainer.java | 2 +- .../maintenance/DeploymentIssueReporter.java | 12 ++++++------ .../maintenance/DeploymentMetricsMaintainer.java | 2 +- .../controller/maintenance/DeploymentUpgrader.java | 2 +- .../maintenance/EnclaveAccessMaintainer.java | 2 +- .../maintenance/EndpointCertificateMaintainer.java | 4 ++-- .../controller/maintenance/HostInfoUpdater.java | 2 +- .../maintenance/InfrastructureUpgrader.java | 4 ++-- .../controller/maintenance/MetricsReporter.java | 2 +- .../controller/maintenance/OsUpgradeScheduler.java | 2 +- .../maintenance/OsVersionStatusUpdater.java | 4 ++-- .../controller/maintenance/ReindexingTriggerer.java | 4 ++-- .../maintenance/ResourceMeterMaintainer.java | 4 ++-- .../maintenance/ResourceTagMaintainer.java | 2 +- .../controller/maintenance/RetriggerMaintainer.java | 4 ++-- .../maintenance/TenantRoleCleanupMaintainer.java | 2 +- .../controller/maintenance/TenantRoleMaintainer.java | 2 +- .../hosted/controller/maintenance/Upgrader.java | 2 +- .../maintenance/UserManagementMaintainer.java | 3 ++- .../controller/maintenance/VcmrMaintainer.java | 2 +- .../controller/maintenance/VersionStatusUpdater.java | 4 ++-- .../controller/maintenance/BcpGroupUpdaterTest.java | 20 ++++++++++---------- .../maintenance/ControllerMaintainerTest.java | 12 ++++++------ .../EndpointCertificateMaintainerTest.java | 16 ++++++++-------- 35 files changed, 103 insertions(+), 84 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 02e1818932e..cbdfcf70123 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -79,7 +79,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to file an issue for '" + application.id() + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private boolean isInCurrentShard(TenantAndApplicationId id) { @@ -122,7 +122,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private double updateConfirmedApplicationOwners() { @@ -149,7 +149,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to find confirmed owner of issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private ApplicationList applications() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 518027f8099..c4f3c611cc5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -98,7 +98,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer { } } - return asSuccessFactor(tenantsByZone.size(), failures); + return asSuccessFactorDeviation(tenantsByZone.size(), failures); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java index 25c4121c271..0b2f8328404 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java @@ -56,10 +56,10 @@ public class ArtifactExpirer extends ControllerMaintainer { log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts in " + cloudName + ": " + artifactsToExpire); artifactRegistry.deleteAll(artifactsToExpire); } - return 1; + return 0; } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to expire artifacts in " + cloudName + ". Will retry in " + interval(), e); - return 0; + return 1; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java index 32e6ad0d557..37fb01fcf92 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java @@ -43,8 +43,8 @@ public class BcpGroupUpdater extends ControllerMaintainer { private final ApplicationController applications; private final NodeRepository nodeRepository; - public BcpGroupUpdater(Controller controller, Duration duration) { - super(controller, duration); + public BcpGroupUpdater(Controller controller, Duration duration, Double successFactorBaseline) { + super(controller, duration, successFactorBaseline); this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); } @@ -58,7 +58,7 @@ public class BcpGroupUpdater extends ControllerMaintainer { for (var application : applications.asList()) { for (var instance : application.instances().values()) { for (var deployment : instance.productionDeployments().values()) { - if (shuttingDown()) return 1.0; + if (shuttingDown()) return 0.0; try { attempts++; var bcpGroups = BcpGroup.groupsFrom(instance, application.deploymentSpec()); @@ -75,12 +75,12 @@ public class BcpGroupUpdater extends ControllerMaintainer { } } } - double successFactor = asSuccessFactor(attempts, failures); - if ( successFactor == 0 ) + double successFactorDeviation = asSuccessFactorDeviation(attempts, failures); + if ( successFactorDeviation == 1.0 ) log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); - else if ( successFactor < 0.9 ) + else if ( successFactorDeviation > 0.1 ) log.log(Level.FINE, "Could not update traffic share on all applications", lastException); - return successFactor; + return successFactorDeviation; } /** Adds deployment traffic share to the given patch. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java index a7ebaec7c09..b40078eef51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java @@ -19,6 +19,6 @@ public class BillingDatabaseMaintainer extends ControllerMaintainer { @Override protected double maintain() { controller().serviceRegistry().billingDatabase().maintain(); - return 1; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java index 914707aa318..68fd5c8bafe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java @@ -19,8 +19,8 @@ public class CloudDatabaseMaintainer extends ControllerMaintainer { controller().serviceRegistry().billingController().updateCache(tenants); } catch (Exception e) { log.warning("Could not update cloud database cache: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 6ecad482cd2..f9c93a87c44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -64,7 +64,7 @@ public class ContactInformationMaintainer extends ControllerMaintainer { interval()); } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java index c861d522818..f21803283eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java @@ -12,7 +12,6 @@ import java.util.EnumSet; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.logging.Logger; /** * A maintainer is some job which runs at a fixed interval to perform some maintenance task in the controller. @@ -26,13 +25,22 @@ public abstract class ControllerMaintainer extends Maintainer { /** The systems in which this maintainer should run */ private final Set activeSystems; + public ControllerMaintainer(Controller controller, Duration interval) { - this(controller, interval, null, EnumSet.allOf(SystemName.class)); + this(controller, interval, null, EnumSet.allOf(SystemName.class), 1.0); + } + + public ControllerMaintainer(Controller controller, Duration interval, Double successFactorBaseline) { + this(controller, interval, null, EnumSet.allOf(SystemName.class), successFactorBaseline); } public ControllerMaintainer(Controller controller, Duration interval, String name, Set activeSystems) { + this(controller, interval, name, activeSystems, 1.0); + } + + public ControllerMaintainer(Controller controller, Duration interval, String name, Set activeSystems, Double successFactorBaseline) { super(name, interval, controller.clock(), controller.jobControl(), - new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true); + new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true, successFactorBaseline); this.controller = controller; this.activeSystems = Set.copyOf(Objects.requireNonNull(activeSystems)); } @@ -54,8 +62,8 @@ public abstract class ControllerMaintainer extends Maintainer { } @Override - public void completed(String job, double successFactor, long durationMs) { - metric.set("maintenance.successFactor", successFactor, metric.createContext(Map.of("job", job))); + public void completed(String job, double successFactorDeviation, long durationMs) { + metric.set("maintenance.successFactorDeviation", successFactorDeviation, metric.createContext(Map.of("job", job))); metric.set("maintenance.duration", durationMs, metric.createContext(Map.of("job", job))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 05159b38ec6..7f48a1115c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -40,6 +40,7 @@ public class ControllerMaintenance extends AbstractComponent { @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) { Intervals intervals = new Intervals(controller.system()); + SuccessFactorBaseline successFactorBaseline = new SuccessFactorBaseline(controller.system()); upgrader = new Upgrader(controller, intervals.defaultInterval); osUpgradeScheduler = new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler); maintainers.add(upgrader); @@ -68,7 +69,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new HostInfoUpdater(controller, intervals.hostInfoUpdater)); maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer)); maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer)); - maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater)); + maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater, successFactorBaseline.trafficFractionUpdater)); maintainers.add(new ArchiveUriUpdater(controller, intervals.archiveUriUpdater)); maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer)); maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); @@ -189,4 +190,15 @@ public class ControllerMaintenance extends AbstractComponent { } + private static class SuccessFactorBaseline { + + private final Double defaultSuccessFactorBaseline; + private final Double trafficFractionUpdater; + + public SuccessFactorBaseline(SystemName system) { + Objects.requireNonNull(system); + this.defaultSuccessFactorBaseline = 1.0; + this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.7; + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 403b5aed1ce..668893d5a7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -34,7 +34,7 @@ public class CostReportMaintainer extends ControllerMaintainer { protected double maintain() { var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations()); consumer.consume(csv); - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 97f3f955a20..c22cb1efdb3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -5,8 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.yolean.Exceptions; @@ -47,7 +45,7 @@ public class DeploymentExpirer extends ControllerMaintainer { } } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } /** Returns whether given deployment has expired according to its TTL */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java index f6029eade37..d8d89177a9e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java @@ -38,7 +38,7 @@ public class DeploymentInfoMaintainer extends ControllerMaintainer { } } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } private Collection instanceDeployments(Instance instance) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 6b058537c2d..c352fb053dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -77,7 +77,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { fileDeploymentIssueFor(application); else store(application.id(), null); - return 1.0; + return 0.0; } /** @@ -87,24 +87,24 @@ public class DeploymentIssueReporter extends ControllerMaintainer { */ private double maintainPlatformIssue(List applications) { if (controller().system() == SystemName.cd) - return 1.0; + return 0.0; VersionStatus versionStatus = controller().readVersionStatus(); Version systemVersion = controller().systemVersion(versionStatus); if (versionStatus.version(systemVersion).confidence() != broken) - return 1.0; + return 0.0; DeploymentStatusList statuses = controller().jobController().deploymentStatuses(ApplicationList.from(applications)); if (statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant().minus(upgradeGracePeriod)).isEmpty()) - return 1.0; + return 0.0; List failingApplications = statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant()) .mapToList(status -> status.application().id().defaultInstance()); // TODO jonmv: Send only tenant and application, here and elsewhere in this. deploymentIssues.fileUnlessOpen(failingApplications, systemVersion); - return 1.0; + return 0.0; } private Tenant ownerOf(TenantAndApplicationId applicationId) { @@ -145,7 +145,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } })); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private void store(TenantAndApplicationId id, IssueId issueId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index fa917d2eb4e..427ee7c4b06 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -96,7 +96,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer { } catch (InterruptedException e) { throw new RuntimeException(e); } - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } static DeploymentMetrics updateDeploymentMetrics(DeploymentMetrics current, List metrics) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index 934a1b4fa2f..8a3a2a11e09 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -78,7 +78,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { ": " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } /** Returns whether query and feed metrics are ~zero, or currently platform has been deployed for a week. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java index 7af96d10f2f..5218da91c46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java @@ -26,7 +26,7 @@ public class EnclaveAccessMaintainer extends ControllerMaintainer { return controller().serviceRegistry().enclaveAccessService().allowAccessFor(externalAccounts()); } catch (RuntimeException e) { logger.log(WARNING, "Failed sharing resources with enclave", e); - return 0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 0b96d8adc1a..713782eb7b9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -75,10 +75,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { deleteOrReportUnmanagedCertificates(); } catch (Exception e) { log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } private void updateRefreshedCertificates() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java index 1f21c688540..31236f4fcda 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java @@ -73,7 +73,7 @@ public class HostInfoUpdater extends ControllerMaintainer { LOG.info("Updated information for " + hostsUpdated + " hosts(s)"); } } - return 1.0; + return 0.0; } private static Optional modelNameOf(NodeEntity nodeEntity) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index b051590ac5a..68c79fdca7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -50,7 +50,7 @@ public abstract class InfrastructureUpgrader exten @Override protected double maintain() { return target().map(target -> upgradeAll(target, managedApplications)) - .orElse(1.0); + .orElse(0.0); } /** Deploy a list of system applications until they converge on the given version */ @@ -81,7 +81,7 @@ public abstract class InfrastructureUpgrader exten break; } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } /** Returns whether all applications have converged to the target version in zone */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 294d5bad42d..67188eb5e3a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -90,7 +90,7 @@ public class MetricsReporter extends ControllerMaintainer { reportBrokenSystemVersion(versionStatus); reportTenantMetrics(); reportZmsQuotaMetrics(); - return 1.0; + return 0.0; } private void reportBrokenSystemVersion(VersionStatus versionStatus) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index f930e64fc5a..f0d218ae6cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -40,7 +40,7 @@ public class OsUpgradeScheduler extends ControllerMaintainer { if (!change.get().scheduleAt(now)) continue; controller().upgradeOsIn(cloud, change.get().version(), false); } - return 1.0; + return 0.0; } /** Returns the wanted change for cloud at given instant, if any */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index 119540eaa68..c643df6af68 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -22,12 +22,12 @@ public class OsVersionStatusUpdater extends ControllerMaintainer { try { OsVersionStatus newStatus = OsVersionStatus.compute(controller()); controller().updateOsVersionStatus(newStatus); - return 1.0; + return 0.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute OS version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return 0.0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java index 400673bfd0c..945b6d32a30 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java @@ -53,11 +53,11 @@ public class ReindexingTriggerer extends ControllerMaintainer { controller().applications().reindex(id, deployment.zone(), List.of(), List.of(), true, speed, "bakground reindexing, to account for changes in built-in linguistics components"); }); - return 1.0; + return 0.0; } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to trigger reindexing: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 3f20c2eac8f..52206d41c00 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -98,13 +98,13 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { } catch (Exception e) { log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots); if (systemName.isPublic()) reportAllScalingEvents(); updateDeploymentCost(resourceSnapshots); - return 1.0; + return 0.0; } void updateDeploymentCost(Collection resourceSnapshots) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index 18ed154fcf1..59871f716e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -43,7 +43,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { if (taggedResources > 0) log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); }); - return 1.0; + return 0.0; } private Map getTenantOfParentHosts(ZoneId zoneId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java index 74bb89e4105..aaf730cc158 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java @@ -47,9 +47,9 @@ public class RetriggerMaintainer extends ControllerMaintainer { controller().curator().writeRetriggerEntries(remaining); } catch (Exception e) { logger.log(Level.WARNING, "Exception while triggering jobs", e); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } /** Returns true if a job is ready to run, i.e. is currently not running */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java index f29df1bc0d5..e3a3415e170 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java @@ -26,6 +26,6 @@ public class TenantRoleCleanupMaintainer extends ControllerMaintainer { controller().serviceRegistry().tenantSecretService().cleanupSecretStores(deletedTenants); } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index bd121871c7c..c7b236880fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -37,7 +37,7 @@ public class TenantRoleMaintainer extends ControllerMaintainer { controller().tenants().updateLastTenantRolesMaintained(t.name(), updated); }); - return 1.0; + return 0.0; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 82b3141e503..edcfcc317a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -64,7 +64,7 @@ public class Upgrader extends ControllerMaintainer { for (UpgradePolicy policy : UpgradePolicy.values()) updateTargets(versionStatus, deploymentStatuses, policy); - return 1.0; + return 0.0; } private DeploymentStatusList deploymentStatuses(VersionStatus versionStatus) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java index 03987efab8b..7c4645a6e48 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java @@ -11,6 +11,7 @@ import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; + /** * Maintains user management resources. * For now, ensures there's no discrepnacy between expected tenant/application roles and auth0/athenz roles @@ -46,7 +47,7 @@ public class UserManagementMaintainer extends ControllerMaintainer { }); } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index da0fa890960..9271f870390 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -90,7 +90,7 @@ public class VcmrMaintainer extends ControllerMaintainer { } }); updateMetrics(); - return 1.0; + return 0.0; } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index 154455c5198..1c4d13aa16d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -39,12 +39,12 @@ public class VersionStatusUpdater extends ControllerMaintainer { controller().serviceRegistry().systemMonitor().reportSystemVersion(version.versionNumber(), convert(version.confidence())); }); - return 1.0; + return 0.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return 0.0; + return 1.0; } static SystemMonitor.Confidence convert(VespaVersion.Confidence confidence) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java index 5deba19c5ea..b8b05bfcfc1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java @@ -45,7 +45,7 @@ public class BcpGroupUpdaterTest { tester.controllerTester().upgradeSystem(Version.fromString("7.1")); var context = tester.newDeploymentContext(); var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1)); - var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1)); + var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0); ZoneId prod1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId prod2 = ZoneId.from("prod", "us-east-3"); ZoneId prod3 = ZoneId.from("prod", "us-west-1"); @@ -57,7 +57,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(50.0, context.application().id().defaultInstance(), prod1, tester); setBcpMetrics(1.5, 0.1, 0.45, context.instanceId(), prod1, "cluster1", tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(1.0, 1.0, context.instanceId(), prod1, tester); assertNoBcpGroupInfo(context.instanceId(), prod1, "cluster1", tester, "No other regions in group"); @@ -67,7 +67,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(20.0, context.application().id().defaultInstance(), prod2, tester); setBcpMetrics(100.0, 0.1, 0.45, context.instanceId(), prod1, "cluster1", tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.75, 1.0, context.instanceId(), prod1, tester); assertTrafficFraction(0.25, 1.0, context.instanceId(), prod2, tester); assertNoBcpGroupInfo(context.instanceId(), prod1, "cluster1", tester, @@ -75,7 +75,7 @@ public class BcpGroupUpdaterTest { assertBcpGroupInfo(100.0, 0.1, 0.45, context.instanceId(), prod2, "cluster1", tester); setBcpMetrics(50.0, 0.2, 0.5, context.instanceId(), prod2, "cluster1", tester); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertBcpGroupInfo(50.0, 0.2, 0.5, context.instanceId(), prod1, "cluster1", tester); @@ -85,7 +85,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(45.0, context.application().id().defaultInstance(), prod2, tester); setQpsMetric(02.0, context.application().id().defaultInstance(), prod3, tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.53, 0.53 + (double)45/2 / 100, context.instanceId(), prod1, tester); assertTrafficFraction(0.45, 0.45 + (double)53/2 / 100, context.instanceId(), prod2, tester); assertTrafficFraction(0.02, 0.02 + (double)53/2 / 100, context.instanceId(), prod3, tester); @@ -129,7 +129,7 @@ public class BcpGroupUpdaterTest { locked -> tester.controller().applications().store(locked.with(deploymentSpec))); var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1)); - var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1)); + var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0); ZoneId ap1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId ap2 = ZoneId.from("prod", "ap-southeast-1"); @@ -150,7 +150,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(40.0, context.application().id().defaultInstance(), eu1, tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.5, 0.5, context.instanceId(), ap1, tester); assertTrafficFraction(0.0, 0.5, context.instanceId(), ap2, tester); assertTrafficFraction(0.1, 0.1, context.instanceId(), us1, tester); @@ -197,7 +197,7 @@ public class BcpGroupUpdaterTest { locked -> tester.controller().applications().store(locked.with(deploymentSpec))); var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1)); - var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1)); + var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0); ZoneId ap1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId ap2 = ZoneId.from("prod", "ap-southeast-1"); @@ -221,7 +221,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(60.0, context.application().id().defaultInstance(), eu1, tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.10, 0.10 + 50 / 200.0 / 1.5, context.instanceId(), ap1, tester); assertTrafficFraction(0.25, 0.25 + 30 / 200.0 / 1.5, context.instanceId(), ap2, tester); assertTrafficFraction(0.00, 0.00 + 40 / 200.0 / 2.5, context.instanceId(), us1, tester); @@ -242,7 +242,7 @@ public class BcpGroupUpdaterTest { setBcpMetrics(300, 0.3, 0.3, context.instanceId(), us3, "cluster2", tester); setBcpMetrics(100, 0.1, 0.1, context.instanceId(), eu1, "cluster2", tester); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertNoBcpGroupInfo(context.instanceId(), ap1, "cluster1", tester, "No info in ap"); assertNoBcpGroupInfo(context.instanceId(), ap2, "cluster1", tester, "No info in ap"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java index 63e2c99cb6e..6452edc9e61 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java @@ -38,14 +38,14 @@ public class ControllerMaintainerTest { void records_metric() { TestControllerMaintainer maintainer = new TestControllerMaintainer(tester.controller(), SystemName.main, new AtomicInteger()); maintainer.run(); - assertEquals(1.0, successFactorMetric(), 0.0000001); + assertEquals(0.0, successFactorDeviationMetric(), 0.0000001); maintainer.success = false; maintainer.run(); maintainer.run(); - assertEquals(0.0, successFactorMetric(), 0.0000001); + assertEquals(1.0, successFactorDeviationMetric(), 0.0000001); maintainer.success = true; maintainer.run(); - assertEquals(1.0, successFactorMetric(), 0.0000001); + assertEquals(0.0, successFactorDeviationMetric(), 0.0000001); } private long consecutiveFailuresMetric() { @@ -54,10 +54,10 @@ public class ControllerMaintainerTest { "maintenance.consecutiveFailures").get().longValue(); } - private long successFactorMetric() { + private long successFactorDeviationMetric() { MetricsMock metrics = (MetricsMock) tester.controller().metric(); return metrics.getMetric((context) -> "TestControllerMaintainer".equals(context.get("job")), - "maintenance.successFactor").get().longValue(); + "maintenance.successFactorDeviation").get().longValue(); } private static class TestControllerMaintainer extends ControllerMaintainer { @@ -73,7 +73,7 @@ public class ControllerMaintainerTest { @Override protected double maintain() { executions.incrementAndGet(); - return success ? 1.0 : 0.0; + return success ? 0.0 : 1.0; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 934e15ad623..49cf8c634ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -49,7 +49,7 @@ public class EndpointCertificateMaintainerTest { @Test void old_and_unused_cert_is_deleted() { tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), exampleMetadata); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertTrue(tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()).isEmpty()); } @@ -57,7 +57,7 @@ public class EndpointCertificateMaintainerTest { void unused_but_recently_used_cert_is_not_deleted() { EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond()); tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(Optional.of(recentlyRequestedCert), tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId())); } @@ -69,7 +69,7 @@ public class EndpointCertificateMaintainerTest { secretStore.setSecret(exampleMetadata.keyName(), "foo", 1); secretStore.setSecret(exampleMetadata.certName(), "bar", 1); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); var updatedCert = Optional.of(recentlyRequestedCert.withLastRefreshed(tester.clock().instant().getEpochSecond()).withVersion(1)); @@ -90,7 +90,7 @@ public class EndpointCertificateMaintainerTest { deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); var metadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow(); tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(metadata.rootRequestId()); // cert should not be deleted, the app is deployed! } @@ -110,7 +110,7 @@ public class EndpointCertificateMaintainerTest { var originalMetadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow(); // cert should not be deleted, the app is deployed! - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(tester.curator().readEndpointCertificateMetadata(appId), Optional.of(originalMetadata)); tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(originalMetadata.rootRequestId()); @@ -121,7 +121,7 @@ public class EndpointCertificateMaintainerTest { tester.controller().serviceRegistry().endpointCertificateProvider().requestCaSignedCertificate(appId, originalMetadata.requestedDnsSans(), Optional.of(originalMetadata)); // We should now pick up the new key and cert version + uuid, but not force trigger deployment yet - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); deploymentContext.assertNotRunning(productionUsWest1); var updatedMetadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow(); assertNotEquals(originalMetadata.leafRequestId().orElseThrow(), updatedMetadata.leafRequestId().orElseThrow()); @@ -130,7 +130,7 @@ public class EndpointCertificateMaintainerTest { // after another 4 days, we should force trigger deployment if it hasn't already happened tester.clock().advance(Duration.ofDays(4).plusSeconds(1)); deploymentContext.assertNotRunning(productionUsWest1); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); deploymentContext.assertRunning(productionUsWest1); } @@ -156,7 +156,7 @@ public class EndpointCertificateMaintainerTest { ApplicationId unknown = ApplicationId.fromSerializedForm("applicationid:is:unknown"); endpointCertificateProvider.requestCaSignedCertificate(unknown, List.of("a", "b", "c"), Optional.empty()); // Unknown to controller! - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertTrue(endpointCertificateProvider.dnsNamesOf(unknown).isEmpty()); assertTrue(endpointCertificateProvider.listCertificates().isEmpty()); -- cgit v1.2.3 From 0880a6cc94fe198bbd022479b3ab6576bb53efab Mon Sep 17 00:00:00 2001 From: bjormel Date: Thu, 30 Mar 2023 14:35:46 +0000 Subject: successFactorDeviation is negative --- .../yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java index 37fb01fcf92..b8c7b5aef9b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java @@ -76,9 +76,9 @@ public class BcpGroupUpdater extends ControllerMaintainer { } } double successFactorDeviation = asSuccessFactorDeviation(attempts, failures); - if ( successFactorDeviation == 1.0 ) + if ( successFactorDeviation == -1.0 ) log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); - else if ( successFactorDeviation > 0.1 ) + else if ( successFactorDeviation < -0.1 ) log.log(Level.FINE, "Could not update traffic share on all applications", lastException); return successFactorDeviation; } -- cgit v1.2.3 From 5353893d4c1c9de95d09b20247d895267e2b5d8c Mon Sep 17 00:00:00 2001 From: bjormel Date: Fri, 31 Mar 2023 12:49:31 +0000 Subject: successFactorDeviation cannot be lower than -successFactorBaseline --- .../yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java index b8c7b5aef9b..fe1930a19ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java @@ -42,11 +42,13 @@ public class BcpGroupUpdater extends ControllerMaintainer { private final ApplicationController applications; private final NodeRepository nodeRepository; + private final Double successFactorBaseline; public BcpGroupUpdater(Controller controller, Duration duration, Double successFactorBaseline) { super(controller, duration, successFactorBaseline); this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + this.successFactorBaseline = successFactorBaseline; } @Override @@ -76,7 +78,7 @@ public class BcpGroupUpdater extends ControllerMaintainer { } } double successFactorDeviation = asSuccessFactorDeviation(attempts, failures); - if ( successFactorDeviation == -1.0 ) + if ( successFactorDeviation == -successFactorBaseline ) log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); else if ( successFactorDeviation < -0.1 ) log.log(Level.FINE, "Could not update traffic share on all applications", lastException); -- cgit v1.2.3