diff options
Diffstat (limited to 'controller-server/src/main')
37 files changed, 106 insertions, 100 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 1eb68c14353..80b52e0c7a4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -31,7 +31,8 @@ import static java.util.Comparator.comparing; */ public class Endpoint { - private static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; + private static final String MAIN_OATH_DNS_SUFFIX = ".vespa.oath.cloud"; + private static final String CD_OATH_DNS_SUFFIX = ".cd.vespa.oath.cloud"; private static final String PUBLIC_DNS_SUFFIX = ".vespa-app.cloud"; private static final String PUBLIC_CD_DNS_SUFFIX = ".cd.vespa-app.cloud"; @@ -243,18 +244,13 @@ public class Endpoint { /** Returns the DNS suffix used for endpoints in given system */ private static String dnsSuffix(SystemName system) { - switch (system) { - case cd, main -> { - return OATH_DNS_SUFFIX; - } - case Public -> { - return PUBLIC_DNS_SUFFIX; - } - case PublicCd -> { - return PUBLIC_CD_DNS_SUFFIX; - } + return switch (system) { + case cd -> CD_OATH_DNS_SUFFIX; + case main -> MAIN_OATH_DNS_SUFFIX; + case Public -> PUBLIC_DNS_SUFFIX; + case PublicCd -> PUBLIC_CD_DNS_SUFFIX; default -> throw new IllegalArgumentException("No DNS suffix declared for system " + system); - } + }; } /** Returns the DNS suffix used for internal names (i.e. names not exposed to tenants) in given system */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java index aa6e3b0c44d..cbd0f685d80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java @@ -49,7 +49,7 @@ public record AuditLog(List<Entry> entries) { public record Entry(Instant at, String principal, Method method, String resource, Optional<String> data, Client client) implements Comparable<Entry> { - private final static int maxDataLength = 1024; + final static int maxDataLength = 1024; private final static Comparator<Entry> comparator = Comparator.comparing(Entry::at).reversed(); public Entry(Instant at, Client client, String principal, Method method, String resource, byte[] data) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java index 033cd0a52c9..13b3d9d170f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java @@ -4,11 +4,12 @@ package com.yahoo.vespa.hosted.controller.auditlog; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.controller.auditlog.AuditLog.Entry; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.UncheckedIOException; +import java.io.InputStream; +import java.io.SequenceInputStream; import java.net.URI; import java.security.Principal; import java.time.Clock; @@ -17,6 +18,9 @@ import java.time.Instant; import java.util.Objects; import java.util.Optional; +import static com.yahoo.yolean.Exceptions.uncheck; +import static java.util.Objects.requireNonNullElse; + /** * This provides read and write operations for the audit log. * @@ -58,14 +62,8 @@ public class AuditLogger { "misconfiguration and should not happen"); } - byte[] data = new byte[0]; - try { - if (request.getData() != null) { - data = request.getData().readAllBytes(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } + InputStream requestData = requireNonNullElse(request.getData(), InputStream.nullInputStream()); + byte[] data = uncheck(() -> requestData.readNBytes(Entry.maxDataLength)); AuditLog.Entry.Client client = parseClient(request); Instant now = clock.instant(); @@ -80,7 +78,9 @@ public class AuditLogger { } // Create a new input stream to allow callers to consume request body - return new HttpRequest(request.getJDiscRequest(), new ByteArrayInputStream(data), request.propertyMap()); + return new HttpRequest(request.getJDiscRequest(), + new SequenceInputStream(new ByteArrayInputStream(data), requestData), + request.propertyMap()); } private static AuditLog.Entry.Client parseClient(HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 050b77a391e..ecd24b3577f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -303,7 +303,7 @@ public class DeploymentStatus { fallbackPlatform(change, job)); if (step.completedAt(change, firstProductionJobWithDeploymentInCloud).isEmpty()) { JobType typeWithZone = job.type().isSystemTest() ? JobType.systemTest(zones, cloud) : JobType.stagingTest(zones, cloud); - jobs.merge(job, List.of(new Job(typeWithZone, versions, step.readyAt(change), change)), DeploymentStatus::union); + jobs.merge(job, List.of(new Job(typeWithZone, versions, step.readyAt(change, firstProductionJobWithDeploymentInCloud), change)), DeploymentStatus::union); } }); }); @@ -679,7 +679,7 @@ public class DeploymentStatus { .asList().isEmpty()) testJobs.merge(testJob, List.of(new Job(testJob.type(), productionJob.versions(), - jobSteps().get(testJob).readyAt(productionJob.change), + jobSteps().get(testJob).readyAt(productionJob.change, Optional.of(job)), productionJob.change)), DeploymentStatus::union); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 02e1818932e..cbdfcf70123 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -79,7 +79,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to file an issue for '" + application.id() + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private boolean isInCurrentShard(TenantAndApplicationId id) { @@ -122,7 +122,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private double updateConfirmedApplicationOwners() { @@ -149,7 +149,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to find confirmed owner of issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private ApplicationList applications() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 518027f8099..c4f3c611cc5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -98,7 +98,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer { } } - return asSuccessFactor(tenantsByZone.size(), failures); + return asSuccessFactorDeviation(tenantsByZone.size(), failures); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java index 25c4121c271..32d06286820 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact; import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry; @@ -13,12 +12,9 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Duration; import java.time.Instant; import java.util.Comparator; -import java.util.EnumSet; import java.util.List; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; /** * Periodically expire unused artifacts, e.g. container images and RPMs. @@ -32,7 +28,7 @@ public class ArtifactExpirer extends ControllerMaintainer { private static final Duration MIN_AGE = Duration.ofDays(14); public ArtifactExpirer(Controller controller, Duration interval) { - super(controller, interval, null, expiringSystems()); + super(controller, interval); } @Override @@ -56,10 +52,10 @@ public class ArtifactExpirer extends ControllerMaintainer { log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts in " + cloudName + ": " + artifactsToExpire); artifactRegistry.deleteAll(artifactsToExpire); } - return 1; + return 0; } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to expire artifacts in " + cloudName + ". Will retry in " + interval(), e); - return 0; + return 1; } } @@ -77,11 +73,4 @@ public class ArtifactExpirer extends ControllerMaintainer { return true; } - /** Returns systems where artifacts can be expired */ - private static Set<SystemName> expiringSystems() { - // Run only in public and main. Public systems have distinct container registries, while main and CD have - // shared registries. - return EnumSet.of(SystemName.Public, SystemName.PublicCd, SystemName.main); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java index 32e6ad0d557..fe1930a19ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java @@ -42,11 +42,13 @@ public class BcpGroupUpdater extends ControllerMaintainer { private final ApplicationController applications; private final NodeRepository nodeRepository; + private final Double successFactorBaseline; - public BcpGroupUpdater(Controller controller, Duration duration) { - super(controller, duration); + public BcpGroupUpdater(Controller controller, Duration duration, Double successFactorBaseline) { + super(controller, duration, successFactorBaseline); this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + this.successFactorBaseline = successFactorBaseline; } @Override @@ -58,7 +60,7 @@ public class BcpGroupUpdater extends ControllerMaintainer { for (var application : applications.asList()) { for (var instance : application.instances().values()) { for (var deployment : instance.productionDeployments().values()) { - if (shuttingDown()) return 1.0; + if (shuttingDown()) return 0.0; try { attempts++; var bcpGroups = BcpGroup.groupsFrom(instance, application.deploymentSpec()); @@ -75,12 +77,12 @@ public class BcpGroupUpdater extends ControllerMaintainer { } } } - double successFactor = asSuccessFactor(attempts, failures); - if ( successFactor == 0 ) + double successFactorDeviation = asSuccessFactorDeviation(attempts, failures); + if ( successFactorDeviation == -successFactorBaseline ) log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); - else if ( successFactor < 0.9 ) + else if ( successFactorDeviation < -0.1 ) log.log(Level.FINE, "Could not update traffic share on all applications", lastException); - return successFactor; + return successFactorDeviation; } /** Adds deployment traffic share to the given patch. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java index a7ebaec7c09..b40078eef51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java @@ -19,6 +19,6 @@ public class BillingDatabaseMaintainer extends ControllerMaintainer { @Override protected double maintain() { controller().serviceRegistry().billingDatabase().maintain(); - return 1; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java index 914707aa318..68fd5c8bafe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java @@ -19,8 +19,8 @@ public class CloudDatabaseMaintainer extends ControllerMaintainer { controller().serviceRegistry().billingController().updateCache(tenants); } catch (Exception e) { log.warning("Could not update cloud database cache: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 6ecad482cd2..f9c93a87c44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -64,7 +64,7 @@ public class ContactInformationMaintainer extends ControllerMaintainer { interval()); } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java index c861d522818..f21803283eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java @@ -12,7 +12,6 @@ import java.util.EnumSet; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.logging.Logger; /** * A maintainer is some job which runs at a fixed interval to perform some maintenance task in the controller. @@ -26,13 +25,22 @@ public abstract class ControllerMaintainer extends Maintainer { /** The systems in which this maintainer should run */ private final Set<SystemName> activeSystems; + public ControllerMaintainer(Controller controller, Duration interval) { - this(controller, interval, null, EnumSet.allOf(SystemName.class)); + this(controller, interval, null, EnumSet.allOf(SystemName.class), 1.0); + } + + public ControllerMaintainer(Controller controller, Duration interval, Double successFactorBaseline) { + this(controller, interval, null, EnumSet.allOf(SystemName.class), successFactorBaseline); } public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems) { + this(controller, interval, name, activeSystems, 1.0); + } + + public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems, Double successFactorBaseline) { super(name, interval, controller.clock(), controller.jobControl(), - new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true); + new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true, successFactorBaseline); this.controller = controller; this.activeSystems = Set.copyOf(Objects.requireNonNull(activeSystems)); } @@ -54,8 +62,8 @@ public abstract class ControllerMaintainer extends Maintainer { } @Override - public void completed(String job, double successFactor, long durationMs) { - metric.set("maintenance.successFactor", successFactor, metric.createContext(Map.of("job", job))); + public void completed(String job, double successFactorDeviation, long durationMs) { + metric.set("maintenance.successFactorDeviation", successFactorDeviation, metric.createContext(Map.of("job", job))); metric.set("maintenance.duration", durationMs, metric.createContext(Map.of("job", job))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 05159b38ec6..7f48a1115c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -40,6 +40,7 @@ public class ControllerMaintenance extends AbstractComponent { @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) { Intervals intervals = new Intervals(controller.system()); + SuccessFactorBaseline successFactorBaseline = new SuccessFactorBaseline(controller.system()); upgrader = new Upgrader(controller, intervals.defaultInterval); osUpgradeScheduler = new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler); maintainers.add(upgrader); @@ -68,7 +69,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new HostInfoUpdater(controller, intervals.hostInfoUpdater)); maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer)); maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer)); - maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater)); + maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater, successFactorBaseline.trafficFractionUpdater)); maintainers.add(new ArchiveUriUpdater(controller, intervals.archiveUriUpdater)); maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer)); maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); @@ -189,4 +190,15 @@ public class ControllerMaintenance extends AbstractComponent { } + private static class SuccessFactorBaseline { + + private final Double defaultSuccessFactorBaseline; + private final Double trafficFractionUpdater; + + public SuccessFactorBaseline(SystemName system) { + Objects.requireNonNull(system); + this.defaultSuccessFactorBaseline = 1.0; + this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.7; + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 403b5aed1ce..668893d5a7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -34,7 +34,7 @@ public class CostReportMaintainer extends ControllerMaintainer { protected double maintain() { var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations()); consumer.consume(csv); - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 97f3f955a20..c22cb1efdb3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -5,8 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.yolean.Exceptions; @@ -47,7 +45,7 @@ public class DeploymentExpirer extends ControllerMaintainer { } } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } /** Returns whether given deployment has expired according to its TTL */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java index f6029eade37..d8d89177a9e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java @@ -38,7 +38,7 @@ public class DeploymentInfoMaintainer extends ControllerMaintainer { } } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } private Collection<DeploymentId> instanceDeployments(Instance instance) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 6b058537c2d..c352fb053dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -77,7 +77,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { fileDeploymentIssueFor(application); else store(application.id(), null); - return 1.0; + return 0.0; } /** @@ -87,24 +87,24 @@ public class DeploymentIssueReporter extends ControllerMaintainer { */ private double maintainPlatformIssue(List<Application> applications) { if (controller().system() == SystemName.cd) - return 1.0; + return 0.0; VersionStatus versionStatus = controller().readVersionStatus(); Version systemVersion = controller().systemVersion(versionStatus); if (versionStatus.version(systemVersion).confidence() != broken) - return 1.0; + return 0.0; DeploymentStatusList statuses = controller().jobController().deploymentStatuses(ApplicationList.from(applications)); if (statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant().minus(upgradeGracePeriod)).isEmpty()) - return 1.0; + return 0.0; List<ApplicationId> failingApplications = statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant()) .mapToList(status -> status.application().id().defaultInstance()); // TODO jonmv: Send only tenant and application, here and elsewhere in this. deploymentIssues.fileUnlessOpen(failingApplications, systemVersion); - return 1.0; + return 0.0; } private Tenant ownerOf(TenantAndApplicationId applicationId) { @@ -145,7 +145,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } })); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private void store(TenantAndApplicationId id, IssueId issueId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index fa917d2eb4e..427ee7c4b06 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -96,7 +96,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer { } catch (InterruptedException e) { throw new RuntimeException(e); } - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } static DeploymentMetrics updateDeploymentMetrics(DeploymentMetrics current, List<ClusterMetrics> metrics) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index 934a1b4fa2f..8a3a2a11e09 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -78,7 +78,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { ": " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } /** Returns whether query and feed metrics are ~zero, or currently platform has been deployed for a week. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java index 7af96d10f2f..5218da91c46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java @@ -26,7 +26,7 @@ public class EnclaveAccessMaintainer extends ControllerMaintainer { return controller().serviceRegistry().enclaveAccessService().allowAccessFor(externalAccounts()); } catch (RuntimeException e) { logger.log(WARNING, "Failed sharing resources with enclave", e); - return 0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 0b96d8adc1a..713782eb7b9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -75,10 +75,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { deleteOrReportUnmanagedCertificates(); } catch (Exception e) { log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } private void updateRefreshedCertificates() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java index 1f21c688540..31236f4fcda 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java @@ -73,7 +73,7 @@ public class HostInfoUpdater extends ControllerMaintainer { LOG.info("Updated information for " + hostsUpdated + " hosts(s)"); } } - return 1.0; + return 0.0; } private static Optional<String> modelNameOf(NodeEntity nodeEntity) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index b051590ac5a..68c79fdca7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -50,7 +50,7 @@ public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> exten @Override protected double maintain() { return target().map(target -> upgradeAll(target, managedApplications)) - .orElse(1.0); + .orElse(0.0); } /** Deploy a list of system applications until they converge on the given version */ @@ -81,7 +81,7 @@ public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> exten break; } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } /** Returns whether all applications have converged to the target version in zone */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 294d5bad42d..67188eb5e3a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -90,7 +90,7 @@ public class MetricsReporter extends ControllerMaintainer { reportBrokenSystemVersion(versionStatus); reportTenantMetrics(); reportZmsQuotaMetrics(); - return 1.0; + return 0.0; } private void reportBrokenSystemVersion(VersionStatus versionStatus) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index f930e64fc5a..f0d218ae6cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -40,7 +40,7 @@ public class OsUpgradeScheduler extends ControllerMaintainer { if (!change.get().scheduleAt(now)) continue; controller().upgradeOsIn(cloud, change.get().version(), false); } - return 1.0; + return 0.0; } /** Returns the wanted change for cloud at given instant, if any */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index 119540eaa68..c643df6af68 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -22,12 +22,12 @@ public class OsVersionStatusUpdater extends ControllerMaintainer { try { OsVersionStatus newStatus = OsVersionStatus.compute(controller()); controller().updateOsVersionStatus(newStatus); - return 1.0; + return 0.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute OS version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return 0.0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java index 400673bfd0c..945b6d32a30 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java @@ -53,11 +53,11 @@ public class ReindexingTriggerer extends ControllerMaintainer { controller().applications().reindex(id, deployment.zone(), List.of(), List.of(), true, speed, "bakground reindexing, to account for changes in built-in linguistics components"); }); - return 1.0; + return 0.0; } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to trigger reindexing: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 3f20c2eac8f..52206d41c00 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -98,13 +98,13 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { } catch (Exception e) { log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots); if (systemName.isPublic()) reportAllScalingEvents(); updateDeploymentCost(resourceSnapshots); - return 1.0; + return 0.0; } void updateDeploymentCost(Collection<ResourceSnapshot> resourceSnapshots) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index 18ed154fcf1..59871f716e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -43,7 +43,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { if (taggedResources > 0) log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); }); - return 1.0; + return 0.0; } private Map<HostName, ApplicationId> getTenantOfParentHosts(ZoneId zoneId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java index 74bb89e4105..aaf730cc158 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java @@ -47,9 +47,9 @@ public class RetriggerMaintainer extends ControllerMaintainer { controller().curator().writeRetriggerEntries(remaining); } catch (Exception e) { logger.log(Level.WARNING, "Exception while triggering jobs", e); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } /** Returns true if a job is ready to run, i.e. is currently not running */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java index f29df1bc0d5..e3a3415e170 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java @@ -26,6 +26,6 @@ public class TenantRoleCleanupMaintainer extends ControllerMaintainer { controller().serviceRegistry().tenantSecretService().cleanupSecretStores(deletedTenants); } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index bd121871c7c..c7b236880fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -37,7 +37,7 @@ public class TenantRoleMaintainer extends ControllerMaintainer { controller().tenants().updateLastTenantRolesMaintained(t.name(), updated); }); - return 1.0; + return 0.0; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 82b3141e503..edcfcc317a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -64,7 +64,7 @@ public class Upgrader extends ControllerMaintainer { for (UpgradePolicy policy : UpgradePolicy.values()) updateTargets(versionStatus, deploymentStatuses, policy); - return 1.0; + return 0.0; } private DeploymentStatusList deploymentStatuses(VersionStatus versionStatus) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java index 03987efab8b..7c4645a6e48 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java @@ -11,6 +11,7 @@ import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; + /** * Maintains user management resources. * For now, ensures there's no discrepnacy between expected tenant/application roles and auth0/athenz roles @@ -46,7 +47,7 @@ public class UserManagementMaintainer extends ControllerMaintainer { }); } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index da0fa890960..9271f870390 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -90,7 +90,7 @@ public class VcmrMaintainer extends ControllerMaintainer { } }); updateMetrics(); - return 1.0; + return 0.0; } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index 154455c5198..1c4d13aa16d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -39,12 +39,12 @@ public class VersionStatusUpdater extends ControllerMaintainer { controller().serviceRegistry().systemMonitor().reportSystemVersion(version.versionNumber(), convert(version.confidence())); }); - return 1.0; + return 0.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return 0.0; + return 1.0; } static SystemMonitor.Confidence convert(VespaVersion.Confidence confidence) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index e9947f3d565..45c00848407 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -41,8 +41,8 @@ public record VespaVersion(Version version, .not().upgradingTo(statistics.version()); InstanceList failingOnThis = all.matching(instance -> statistics.failingUpgrades().stream().anyMatch(run -> run.id().application().equals(instance))); - // 'broken' if any canary fails - if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty()) + // 'broken' if any canary fails, and no non-canary is upgraded + if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty() && productionOnThis.not().with(UpgradePolicy.canary).isEmpty()) return Confidence.broken; // 'broken' if 6 non-canary was broken by this, and that is at least 5% of all |