diff options
Diffstat (limited to 'controller-server')
38 files changed, 570 insertions, 96 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index eb8f6ed6ae2..1c4f0994f8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.component.Vtag; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java index 6b93229dffe..c88eb2f1b86 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.impl; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.client.ErrorHandler; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 2ad28893e18..e4869cc9bf4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index cc7031bab5a..52283a3e27d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -211,6 +211,11 @@ public class DeploymentStatus { /** The set of jobs that need to run for the given changes to be considered complete. */ public boolean hasCompleted(InstanceName instance, Change change) { + if ( ! application.deploymentSpec().requireInstance(instance).concerns(prod)) { + if (newestTested(instance, run -> run.versions().targetRevision()).map(change::downgrades).orElse(false)) return true; + if (newestTested(instance, run -> run.versions().targetPlatform()).map(change::downgrades).orElse(false)) return true; + } + return jobsToRun(Map.of(instance, change), false).isEmpty(); } @@ -247,6 +252,14 @@ public class DeploymentStatus { .deployments().get(job.type().zone())); } + private <T extends Comparable<T>> Optional<T> newestTested(InstanceName instance, Function<Run, T> runMapper) { + return instanceJobs().get(application.id().instance(instance)) + .type(systemTest, stagingTest) + .asList().stream().flatMap(jobs -> jobs.runs().values().stream()) + .filter(Run::hasSucceeded) + .map(runMapper) + .max(naturalOrder()); + } /** * The change to a revision which all dependencies of the given instance has completed, * which does not downgrade any deployments in the instance, @@ -263,16 +276,20 @@ public class DeploymentStatus { int nextRisk = 0; int skippedCumulativeRisk = 0; Instant readySince = now; + + Optional<RevisionId> newestRevision = application.productionDeployments() + .getOrDefault(instance, List.of()).stream() + .map(Deployment::revision).max(naturalOrder()); Change candidate = Change.empty(); for (ApplicationVersion version : application.revisions().deployable(ascending)) { // A revision is only a candidate if it upgrades, and does not downgrade, this instance. Change change = Change.of(version.id()); - if (application.productionDeployments().getOrDefault(instance, List.of()).stream() - .anyMatch(deployment -> change.downgrades(deployment.revision()))) continue; - if ( ! application.require(instance).change().revision().map(change::upgrades).orElse(true)) continue; - if (hasCompleted(instance, change)) + if ( newestRevision.isPresent() && change.downgrades(newestRevision.get()) + || ! application.require(instance).change().revision().map(change::upgrades).orElse(true) + || hasCompleted(instance, change)) { if (ascending) continue; // Keep looking for the next revision which is an upgrade, or ... else return Change.empty(); // ... if the latest is already complete, there's nothing outstanding. + } // This revision contains something new, so start aggregating the risk score. skippedCumulativeRisk += version.risk(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 1d56e2db08b..30f16acf77d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; @@ -40,9 +39,10 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; +import java.util.ArrayDeque; import java.util.Collections; import java.util.Comparator; +import java.util.Deque; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -55,7 +55,6 @@ import java.util.TreeMap; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.logging.Logger; @@ -390,12 +389,13 @@ public class JobController { * Throws TimeoutException if some step in this job is still being run. */ public void finish(RunId id) throws TimeoutException { - List<Mutex> locks = new ArrayList<>(); + Deque<Mutex> locks = new ArrayDeque<>(); try { // Ensure no step is still running before we finish the run — report depends transitively on all the other steps. Run unlockedRun = run(id).get(); + locks.push(curator.lock(id.application(), id.type(), report)); for (Step step : report.allPrerequisites(unlockedRun.steps().keySet())) - locks.add(curator.lock(id.application(), id.type(), step)); + locks.push(curator.lock(id.application(), id.type(), step)); locked(id, run -> { // If run should be reset, just return here. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 82d154dcf03..379cc9c4f0a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -5,6 +5,7 @@ import java.util.Collection; import java.util.List; import java.util.stream.Stream; +import static java.util.Comparator.reverseOrder; import static java.util.stream.Collectors.toList; /** @@ -87,7 +88,7 @@ public enum Step { .filter(among::contains) .flatMap(pre -> Stream.concat(Stream.of(pre), pre.allPrerequisites(among).stream())) - .sorted() + .sorted(reverseOrder()) .distinct() .collect(toList()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index e929940b68b..193d171e334 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.concurrent.maintenance.Maintainer; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 193fb89eb99..f3256237284 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Sets; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 4ed34a91029..9cddbd0b903 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -101,7 +101,8 @@ public class Upgrader extends ControllerMaintainer { cancelUpgradesOf(outdated.upgrading(), "Upgrading to outdated versions"); // Prefer the newest target for each instance. - remaining = remaining.not().matching(eligible.asList()::contains); + remaining = remaining.not().matching(eligible.asList()::contains) + .not().hasCompleted(Change.of(version)); for (ApplicationId id : outdated.and(eligible.not().upgrading()).not().changingRevision()) targets.put(id, version); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index 1463cce595d..551f803f368 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -214,6 +214,10 @@ public class VcmrMaintainer extends ControllerMaintainer { return hostAction.withState(State.PENDING_RETIREMENT); } + if (isFailed(node)) { + return hostAction.withState(State.NONE); + } + return hostAction; } @@ -260,6 +264,11 @@ public class VcmrMaintainer extends ControllerMaintainer { action.getState() == State.RETIRING && !node.wantToRetire(); } + private boolean isFailed(Node node) { + return node.state() == Node.State.failed || + node.state() == Node.State.breakfixed; + } + private Map<ZoneId, List<Node>> nodesByZone() { return controller().zoneRegistry() .zones() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java new file mode 100644 index 00000000000..8a6243a7224 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java @@ -0,0 +1,40 @@ +package com.yahoo.vespa.hosted.controller.notification; + +import java.net.URI; +import java.util.Objects; + +/** + * Contains formatted text that can be displayed to a user to give extra information and pointers for a given + * Notification. + * + * @author enygaard + */ +public class FormattedNotification { + private final String prettyType; + private final String messagePrefix; + private final URI uri; + private final Notification notification; + + public FormattedNotification(Notification notification, String prettyType, String messagePrefix, URI uri) { + this.prettyType = Objects.requireNonNull(prettyType); + this.messagePrefix = Objects.requireNonNull(messagePrefix); + this.uri = Objects.requireNonNull(uri); + this.notification = Objects.requireNonNull(notification); + } + + public String prettyType() { + return prettyType; + } + + public String messagePrefix() { + return messagePrefix; + } + + public URI uri() { + return uri; + } + + public Notification notification() { + return notification; + } +}
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MissingOptionalException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MissingOptionalException.java new file mode 100644 index 00000000000..1379ab4654f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MissingOptionalException.java @@ -0,0 +1,18 @@ +package com.yahoo.vespa.hosted.controller.notification; + +/** + * Used to signal that an expected value was not present when creating NotificationContent + * + * @author enygaard + */ +class MissingOptionalException extends RuntimeException { + private final String field; + public MissingOptionalException(String field) { + super(field + " was expected but not present"); + this.field = field; + } + + public String field() { + return field; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java new file mode 100644 index 00000000000..d2b12ab6edc --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java @@ -0,0 +1,190 @@ +package com.yahoo.vespa.hosted.controller.notification; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.text.Text; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import org.apache.http.client.utils.URIBuilder; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; + +/** + * Created a NotificationContent for a given Notification. + * + * The formatter will create specific summary, message start and URI for a given Notification. + * + * @author enygaard + */ +public class NotificationFormatter { + private final ZoneRegistry zoneRegistry; + + public NotificationFormatter(ZoneRegistry zoneRegistry) { + this.zoneRegistry = Objects.requireNonNull(zoneRegistry); + } + + public FormattedNotification format(Notification n) { + switch (n.type()) { + case applicationPackage: + case submission: + return applicationPackage(n); + case deployment: + return deployment(n); + case testPackage: + return testPackage(n); + case reindex: + return reindex(n); + case feedBlock: + return feedBlock(n); + default: + return new FormattedNotification(n, n.type().name(), "", zoneRegistry.dashboardUrl(n.source().tenant())); + } + } + + private FormattedNotification applicationPackage(Notification n) { + var source = n.source(); + var application = requirePresent(source.application(), "application"); + var instance = requirePresent(source.instance(), "instance"); + var message = Text.format("Application package for %s.%s has %s", + application, + instance, + levelText(n.level(), n.messages().size())); + var uri = zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), application, instance)); + return new FormattedNotification(n, "Application package", message, uri); + } + + private FormattedNotification deployment(Notification n) { + var source = n.source(); + var message = Text.format("%s for %s.%s has %s", + jobText(source), + requirePresent(source.application(), "application"), + requirePresent(source.instance(), "instance"), + levelText(n.level(), n.messages().size())); + return new FormattedNotification(n,"Deployment", message, jobLink(n.source())); + } + + private FormattedNotification testPackage(Notification n) { + var source = n.source(); + var application = requirePresent(source.application(), "application"); + var message = Text.format("There %s with tests for %s%s", + n.messages().size() > 1 ? "are problems" : "is a problem", + application, + source.instance().map(i -> "."+i).orElse("")); + var uri = zoneRegistry.dashboardUrl(source.tenant(), application); + return new FormattedNotification(n, "Test package", message, uri); + } + + private FormattedNotification reindex(Notification n) { + var message = Text.format("%s is reindexing", clusterInfo(n.source())); + var source = n.source(); + var application = requirePresent(source.application(), "application"); + var instance = requirePresent(source.instance(), "instance"); + var clusterId = requirePresent(source.clusterId(), "clusterId"); + var zone = requirePresent(source.zoneId(), "zoneId"); + var instanceURI = zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), application, instance)); + try { + var uri = new URIBuilder(instanceURI) + .setParameter( + String.format("%s.%s.%s", instance, zone.environment(), zone.region()), + String.format("clusters,%s=status", clusterId.value())) + .build(); + return new FormattedNotification(n, "Reindex", message, uri); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + + private FormattedNotification feedBlock(Notification n) { + String type; + if (n.level() == Notification.Level.warning) { + type = "Nearly feed blocked"; + } else { + type = "Feed blocked"; + } + var message = Text.format("%s is %s", clusterInfo(n.source()), type.toLowerCase()); + var source = n.source(); + var application = requirePresent(source.application(), "application"); + var instance = requirePresent(source.instance(), "instance"); + var clusterId = requirePresent(source.clusterId(), "clusterId"); + var zone = requirePresent(source.zoneId(), "zoneId"); + var instanceURI = zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), application, instance)); + try { + var uri = new URIBuilder(instanceURI) + .setParameter( + String.format("%s.%s.%s", instance, zone.environment(), zone.region()), + String.format("clusters,%s", clusterId.value())) + .build(); + return new FormattedNotification(n, type, message, uri); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + + private URI jobLink(NotificationSource source) { + var application = requirePresent(source.application(), "application"); + var instance = requirePresent(source.instance(), "instance"); + var jobType = requirePresent(source.jobType(), "jobType"); + var runNumber = source.runNumber().orElseThrow(() -> new MissingOptionalException("runNumber")); + var applicationId = ApplicationId.from(source.tenant(), application, instance); + Function<Environment, URI> link = (Environment env) -> zoneRegistry.dashboardUrl(new RunId(applicationId, jobType, runNumber)); + var environment = jobType.zone().environment(); + switch (environment) { + case dev: + case perf: + return link.apply(environment); + default: + return link.apply(Environment.prod); + } + } + + private String jobText(NotificationSource source) { + var jobType = requirePresent(source.jobType(), "jobType"); + var zone = jobType.zone(); + var runNumber = source.runNumber().orElseThrow(() -> new MissingOptionalException("runNumber")); + switch (zone.environment().value()) { + case "production": + return Text.format("Deployment job #%d to %s", runNumber, zone.region()); + case "test": + return Text.format("Test job #%d to %s", runNumber, zone.region()); + case "dev": + case "perf": + return Text.format("Deployment job #%d to %s.%s", runNumber, zone.environment().value(), zone.region().value()); + } + switch (jobType.jobName()) { + case "system-test": + case "staging-test": + } + return Text.format("%s #%d", jobType.jobName(), runNumber); + } + + private String levelText(Notification.Level level, int count) { + switch (level) { + case error: + return "failed"; + case warning: + return count > 1 ? Text.format("%d warnings", count) : "a warning"; + default: + return count > 1 ? Text.format("%d messages", count) : "a message"; + } + } + + private String clusterInfo(NotificationSource source) { + var application = requirePresent(source.application(), "application"); + var instance = requirePresent(source.instance(), "instance"); + var zone = requirePresent(source.zoneId(), "zoneId"); + var clusterId = requirePresent(source.clusterId(), "clusterId"); + return Text.format("Cluster %s in %s.%s for %s.%s", + clusterId.value(), + zone.environment(), zone.region(), + application, instance); + } + + + private static <T> T requirePresent(Optional<T> optional, String field) { + return optional.orElseThrow(() -> new MissingOptionalException(field)); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java index b098b779dbd..5a5188da37f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java @@ -1,13 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.text.Text; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; @@ -16,7 +14,6 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; -import java.net.URI; import java.util.Collection; import java.util.List; import java.util.Objects; @@ -31,17 +28,17 @@ import java.util.stream.Collectors; */ public class Notifier { private final CuratorDb curatorDb; - private final ZoneRegistry zoneRegistry; private final Mailer mailer; private final FlagSource flagSource; + private final NotificationFormatter formatter; private static final Logger log = Logger.getLogger(Notifier.class.getName()); public Notifier(CuratorDb curatorDb, ZoneRegistry zoneRegistry, Mailer mailer, FlagSource flagSource) { this.curatorDb = Objects.requireNonNull(curatorDb); - this.zoneRegistry = Objects.requireNonNull(zoneRegistry); this.mailer = Objects.requireNonNull(mailer); this.flagSource = Objects.requireNonNull(flagSource); + this.formatter = new NotificationFormatter(zoneRegistry); } public void dispatch(List<Notification> notifications, NotificationSource source) { @@ -64,6 +61,10 @@ public class Notifier { }); } + public void dispatch(Notification notification) { + dispatch(List.of(notification), notification.source()); + } + private boolean dispatchEnabled(NotificationSource source) { return Flags.NOTIFICATION_DISPATCH_FLAG.bindTo(flagSource) .with(FetchVector.Dimension.TENANT_ID, source.tenant().value()) @@ -80,10 +81,6 @@ public class Notifier { return false; } - public void dispatch(Notification notification) { - dispatch(List.of(notification), notification.source()); - } - private void dispatch(Notification notification, TenantContacts.Type type, Collection<? extends TenantContacts.Contact> contacts) { switch (type) { case EMAIL: @@ -96,21 +93,24 @@ public class Notifier { private void dispatch(Notification notification, Collection<TenantContacts.EmailContact> contacts) { try { - mailer.send(mailOf(notification, contacts.stream().map(c -> c.email()).collect(Collectors.toList()))); + var content = formatter.format(notification); + mailer.send(mailOf(content, contacts.stream().map(c -> c.email()).collect(Collectors.toList()))); } catch (MailerException e) { log.log(Level.SEVERE, "Failed sending email", e); + } catch (MissingOptionalException e) { + log.log(Level.WARNING, "Missing value in required field '" + e.field() + "' for notification type: " + notification.type(), e); } } - private Mail mailOf(Notification n, Collection<String> recipients) { - var source = n.source(); - var subject = Text.format("[%s] %s Vespa Notification for %s", n.level().toString().toUpperCase(), n.type().name(), applicationIdSource(source)); + private Mail mailOf(FormattedNotification content, Collection<String> recipients) { + var notification = content.notification(); + var subject = Text.format("[%s] %s Vespa Notification for %s", notification.level().toString().toUpperCase(), content.prettyType(), applicationIdSource(notification.source())); var body = new StringBuilder(); - body.append("Source: ").append(n.source().toString()).append("\n") - .append("\n") - .append(String.join("\n", n.messages())) + body.append(content.messagePrefix()).append("\n\n") + .append(notification.messages().stream().map(m -> " * " + m).collect(Collectors.joining("\n"))).append("\n") .append("\n") - .append(url(source).toString()); + .append("Vespa Console link:\n") + .append(content.uri().toString()); return new Mail(recipients, subject, body.toString()); } @@ -122,22 +122,5 @@ public class Notifier { return sb.toString(); } - private URI url(NotificationSource source) { - if (source.application().isPresent()) { - if (source.instance().isPresent()) { - if (source.jobType().isPresent() && source.runNumber().isPresent()) { - return zoneRegistry.dashboardUrl( - new RunId(ApplicationId.from(source.tenant(), - source.application().get(), - source.instance().get()), - source.jobType().get(), - source.runNumber().getAsLong())); - } - return zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), source.application().get(), source.instance().get())); - } - return zoneRegistry.dashboardUrl(source.tenant(), source.application().get()); - } - return zoneRegistry.dashboardUrl(source.tenant()); - } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index c3c68f7596f..f02f49e7114 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.concurrent.UncheckedTimeoutException; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java index 21414339a87..8f36daf9756 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.curator.mock.MockCurator; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index dffc7bf43b4..671222e2123 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.text.Text; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 41055339ac6..e50f81919df 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -10,7 +10,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; @@ -29,6 +29,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.restapi.ByteArrayResponse; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; @@ -72,7 +73,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; @@ -1120,6 +1120,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); nodeObject.setString("group", node.group()); + nodeObject.setLong("index", node.index()); } return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java index 9e218ce89e6..56d82d286cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.athenz; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.SystemName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.ResourceResponse; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 01bd02fdc13..4aefb9ea7c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import com.auth0.jwt.JWT; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 7c695ef51d7..cef6840dfe1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.DiscFilterRequest; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java index 4762d2c4612..e840b70a95a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index b7c77e7bfb4..5eaa6d7af1d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import ai.vespa.hosted.api.Method; import ai.vespa.hosted.api.RequestVerifier; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java index 708891c8251..f9f3025837d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.horizon; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index e0097c295c7..aaaf09fa781 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.systemflags; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 025e8dff659..fce2d283da2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.user; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java index f4f6df28ebc..4ece2b9a691 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.slime.Inspector; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index 39ee2a6ce44..87691d2927a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index a08ec77bbb9..9bf0f8dcde2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tls; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; import com.yahoo.security.KeyStoreBuilder; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index fd294f9cf9f..ae4b6259da1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; -import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; -import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; @@ -20,19 +19,20 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner.Timeouts; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; @@ -50,9 +50,11 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; @@ -110,7 +112,7 @@ public class DeploymentContext { .parallel("us-west-1", "us-east-3") .emailRole("author") .emailAddress("b@a") - .build()); + .build())::get; private static final Supplier<ApplicationPackage> publicCdApplicationPackage = Suppliers.memoize(() -> new ApplicationPackageBuilder() .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) @@ -119,7 +121,7 @@ public class DeploymentContext { .emailRole("author") .emailAddress("b@a") .trust(generateCertificate()) - .build()); + .build())::get; public static final SourceRevision defaultSourceRevision = new SourceRevision("repository1", "master", "commit1"); @@ -268,7 +270,7 @@ public class DeploymentContext { /** Add a routing policy for this in given zone, with status set to inactive */ public DeploymentContext addInactiveRoutingPolicy(ZoneId zone) { var clusterId = "default-inactive"; - var id = new RoutingPolicyId(instanceId, ClusterSpec.Id.from(clusterId), zone); + var id = new RoutingPolicyId(instanceId, Id.from(clusterId), zone); var policies = new LinkedHashMap<>(tester.controller().routing().policies().read(instanceId).asMap()); policies.put(id, new RoutingPolicy(id, HostName.of("lb-host"), Optional.empty(), @@ -330,7 +332,7 @@ public class DeploymentContext { /** Fail current deployment in given job */ public DeploymentContext nodeAllocationFailure(JobType type) { return failDeployment(type, - new ConfigServerException(ConfigServerException.ErrorCode.NODE_ALLOCATION_FAILURE, + new ConfigServerException(ErrorCode.NODE_ALLOCATION_FAILURE, "Node allocation failure", "Failed to deploy application")); } @@ -359,7 +361,7 @@ public class DeploymentContext { if (messagePart.isPresent()) { Optional<Step> firstFailing = run.stepStatuses().entrySet().stream() .filter(kv -> kv.getValue() == failed) - .map(Map.Entry::getKey) + .map(Entry::getKey) .findFirst(); assertTrue("Found failing step", firstFailing.isPresent()); Optional<RunLog> details = jobs.details(id); @@ -490,7 +492,7 @@ public class DeploymentContext { triggerJobs(); RunId id = currentRun(job).id(); doDeploy(job); - tester.clock().advance(InternalStepRunner.Timeouts.of(tester.controller().system()).noNodesDown().plusSeconds(1)); + tester.clock().advance(Timeouts.of(tester.controller().system()).noNodesDown().plusSeconds(1)); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); assertTrue(jobs.run(id).get().hasEnded()); @@ -504,7 +506,7 @@ public class DeploymentContext { RunId id = currentRun(job).id(); doDeploy(job); doUpgrade(job); - tester.clock().advance(InternalStepRunner.Timeouts.of(tester.controller().system()).noNodesDown().plusSeconds(1)); + tester.clock().advance(Timeouts.of(tester.controller().system()).noNodesDown().plusSeconds(1)); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); assertTrue(jobs.run(id).get().hasEnded()); @@ -574,15 +576,15 @@ public class DeploymentContext { tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), tester.configServer().application(job.application(), zone).get().version().get()); configServer().convergeServices(id.application(), zone); runner.advance(currentRun(job)); - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); // All installation is complete and endpoints are ready, so setup may begin. - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.startStagingSetup)); + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.startStagingSetup)); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endStagingSetup)); - tester.cloud().set(TesterCloud.Status.SUCCESS); + tester.cloud().set(Status.SUCCESS); runner.advance(currentRun(job)); assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.endStagingSetup)); } @@ -618,11 +620,11 @@ public class DeploymentContext { configServer().convergeServices(id.application(), zone); runner.advance(currentRun(job)); if (job.type().environment().isManuallyDeployed()) { - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); assertTrue(jobs.run(id).get().hasEnded()); return; } - assertEquals("Status of " + id, Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); + assertEquals("Status of " + id, succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); } /** Installs tester and starts tests. */ @@ -647,12 +649,12 @@ public class DeploymentContext { // All installation is complete and endpoints are ready, so tests may begin. if (job.type().isDeployment()) - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.startTests)); + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.startTests)); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests)); - tester.cloud().set(TesterCloud.Status.SUCCESS); + tester.cloud().set(Status.SUCCESS); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasEnded()); assertFalse(jobs.run(id).get().hasFailed()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 4d4d94f9e1f..beda9bd551d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.Change; @@ -2147,4 +2148,123 @@ public class DeploymentTriggerTest { assertEquals(version2, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); } + @Test + public void testInstanceWithOnlySystemTest() { + String spec = "<deployment>\n" + + " <instance id='tests'>" + + " <test />\n" + + " <upgrade revision-target='next' />" + + " </instance>\n" + + " <instance id='main'>\n" + + " <prod>\n" + + " <region>us-east-3</region>\n" + + " </prod>\n" + + " <upgrade revision-target='next' />" + + " </instance>\n" + + "</deployment>\n"; + ApplicationPackage appPackage = ApplicationPackageBuilder.fromDeploymentXml(spec); + DeploymentContext tests = tester.newDeploymentContext("tenant", "application", "tests"); + DeploymentContext main = tester.newDeploymentContext("tenant", "application", "main"); + Version version1 = new Version("7"); + tester.controllerTester().upgradeSystem(version1); + tests.submit(appPackage).deploy(); + Optional<RevisionId> revision1 = tests.lastSubmission(); + JobId systemTestJob = new JobId(tests.instanceId(), systemTest); + JobId stagingTestJob = new JobId(tests.instanceId(), stagingTest); + JobId mainJob = new JobId(main.instanceId(), productionUsEast3); + + assertEquals(Change.empty(), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); + + // Versions 2 and 3 become available. + // Tests instance fails on 2, then update to 3. + // Version 2 should not be a target for either instance. + // Version 2 should also not be possible to set as a forced target for the tests instance. + Version version2 = new Version("8"); + tester.controllerTester().upgradeSystem(version2); + tester.upgrader().run(); + tester.triggerJobs(); + + assertEquals(Change.of(version2), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); + + Version version3 = new Version("9"); + tester.controllerTester().upgradeSystem(version3); + tests.failDeployment(systemTest); + tester.upgrader().run(); + + assertEquals(Change.of(version3), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); + + tests.runJob(systemTest); + tester.upgrader().run(); + tests.runJob(stagingTest); + + assertEquals(Change.empty(), tests.instance().change()); + assertEquals(Change.of(version3), main.instance().change()); + assertEquals(Set.of(mainJob), tests.deploymentStatus().jobsToRun().keySet()); + + main.runJob(productionUsEast3); + + assertEquals(Change.empty(), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); + + tester.deploymentTrigger().forceChange(tests.instanceId(), Change.of(version2)); + + assertEquals(Change.empty(), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); + + // Revisions 2 and 3 become available. + // Tests instance fails on 2, then update to 3. + // Revision 2 should not be a target for either instance. + // Revision 2 should also not be possible to set as a forced target for the tests instance. + tests.submit(appPackage); + Optional<RevisionId> revision2 = tests.lastSubmission(); + + assertEquals(Change.of(revision2.get()), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); + + tests.submit(appPackage); + Optional<RevisionId> revision3 = tests.lastSubmission(); + + assertEquals(Change.of(revision2.get()), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); + + tests.failDeployment(systemTest); + tester.outstandingChangeDeployer().run(); + + assertEquals(Change.of(revision3.get()), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); + + tests.runJob(systemTest); + tester.outstandingChangeDeployer().run(); + tester.outstandingChangeDeployer().run(); + tests.runJob(stagingTest); + + assertEquals(Change.empty(), tests.instance().change()); + assertEquals(Change.of(revision3.get()), main.instance().change()); + assertEquals(Set.of(mainJob), tests.deploymentStatus().jobsToRun().keySet()); + + main.runJob(productionUsEast3); + tester.outstandingChangeDeployer().run(); + + assertEquals(Change.empty(), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); + + tester.deploymentTrigger().forceChange(tests.instanceId(), Change.of(revision2.get())); + + assertEquals(Change.empty(), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 1ed84659f58..b3689dacea7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.integration; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL.Path; import ai.vespa.http.HttpURL.Query; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index d923a4d1207..a91d81bb0c5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 185c1e8c891..3b21d3a017e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -905,10 +905,9 @@ public class UpgraderTest { tester.controllerTester().upgradeSystem(version0); // Create an application with pinned platform version. - var context = tester.newDeploymentContext(); + var context = tester.newDeploymentContext().submit().deploy(); tester.deploymentTrigger().forceChange(context.instanceId(), Change.empty().withPin()); - context.submit().deploy(); assertFalse(context.instance().change().hasTargets()); assertTrue(context.instance().change().isPinned()); assertEquals(3, context.instance().deployments().size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatterTest.java new file mode 100644 index 00000000000..c643a612f00 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatterTest.java @@ -0,0 +1,92 @@ +package com.yahoo.vespa.hosted.controller.notification; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; +import org.junit.Test; + +import java.time.Instant; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author enygaard + */ +public class NotificationFormatterTest { + private final TenantName tenant = TenantName.from("scoober"); + private final ApplicationName application = ApplicationName.from("myapp"); + private final InstanceName instance = InstanceName.from("beta"); + private final ApplicationId applicationId = ApplicationId.from(tenant, application, instance); + private final DeploymentId deploymentId = new DeploymentId(applicationId, ZoneId.defaultId()); + private final ClusterSpec.Id cluster = new ClusterSpec.Id("content"); + private final ZoneRegistryMock zoneRegistry = new ZoneRegistryMock(SystemName.Public); + + private final NotificationFormatter formatter = new NotificationFormatter(zoneRegistry); + + @Test + public void applicationPackage() { + var notification = new Notification(Instant.now(), Notification.Type.applicationPackage, Notification.Level.warning, NotificationSource.from(applicationId), List.of("1", "2")); + var content = formatter.format(notification); + assertEquals("Application package", content.prettyType()); + assertEquals("Application package for myapp.beta has 2 warnings", content.messagePrefix()); + assertEquals("https://dashboard.tld/scoober.myapp.beta", content.uri().toString()); + } + + @Test + public void deployment() { + var runId = new RunId(applicationId, JobType.prod(RegionName.defaultName()), 1001); + var notification = new Notification(Instant.now(), Notification.Type.deployment, Notification.Level.warning, NotificationSource.from(runId), List.of("1")); + var content = formatter.format(notification); + assertEquals("Deployment", content.prettyType()); + assertEquals("production-default #1001 for myapp.beta has a warning", content.messagePrefix()); + assertEquals("https://dashboard.tld/scoober.myapp.beta/production-default/1001", content.uri().toString()); + } + + @Test + public void deploymentError() { + var runId = new RunId(applicationId, JobType.prod(RegionName.defaultName()), 1001); + var notification = new Notification(Instant.now(), Notification.Type.deployment, Notification.Level.error, NotificationSource.from(runId), List.of("1")); + var content = formatter.format(notification); + assertEquals("Deployment", content.prettyType()); + assertEquals("production-default #1001 for myapp.beta has failed", content.messagePrefix()); + assertEquals("https://dashboard.tld/scoober.myapp.beta/production-default/1001", content.uri().toString()); + } + + @Test + public void testPackage() { + var notification = new Notification(Instant.now(), Notification.Type.testPackage, Notification.Level.warning, NotificationSource.from(TenantAndApplicationId.from(applicationId)), List.of("1")); + var content = formatter.format(notification); + assertEquals("Test package", content.prettyType()); + assertEquals("There is a problem with tests for myapp", content.messagePrefix()); + assertEquals("https://dashboard.tld/scoober/myapp", content.uri().toString()); + } + + @Test + public void reindex() { + var notification = new Notification(Instant.now(), Notification.Type.reindex, Notification.Level.info, NotificationSource.from(deploymentId, cluster), List.of("1")); + var content = formatter.format(notification); + assertEquals("Reindex", content.prettyType()); + assertEquals("Cluster content in prod.default for myapp.beta is reindexing", content.messagePrefix()); + assertEquals("https://dashboard.tld/scoober.myapp.beta?beta.prod.default=clusters%2Ccontent%3Dstatus", content.uri().toString()); + } + + @Test + public void feedBlock() { + var notification = new Notification(Instant.now(), Notification.Type.feedBlock, Notification.Level.warning, NotificationSource.from(deploymentId, cluster), List.of("1")); + var content = formatter.format(notification); + assertEquals("Nearly feed blocked", content.prettyType()); + assertEquals("Cluster content in prod.default for myapp.beta is nearly feed blocked", content.messagePrefix()); + assertEquals("https://dashboard.tld/scoober.myapp.beta?beta.prod.default=clusters%2Ccontent", content.uri().toString()); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index edbee6e3900..75dbebe96ff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -104,9 +104,9 @@ public class NotificationsDbTest { @Test public void notifier_test() { Notification notification1 = notification(12345, Type.deployment, Level.warning, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg #2"); - Notification notification2 = notification(12345, Type.deployment, Level.error, NotificationSource.from(ApplicationId.from(tenant.value(), "app3", "instance2")), "instance msg #3"); - Notification notification3 = notification(12345, Type.reindex, Level.warning, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg #2"); - + Notification notification2 = notification(12345, Type.applicationPackage, Level.error, NotificationSource.from(ApplicationId.from(tenant.value(), "app3", "instance2")), "instance msg #3"); + Notification notification3 = notification(12345, Type.reindex, Level.warning, NotificationSource.from(new DeploymentId(ApplicationId.from(tenant.value(), "app2", "instance2"), ZoneId.defaultId()), new ClusterSpec.Id("content")), "instance msg #2"); +; var a = notifications.get(0); notificationsDb.setNotification(a.source(), a.type(), a.level(), a.messages()); assertEquals(0, mailer.inbox(email).size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json index 88a92bf7810..34d2f054e0f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json @@ -18,7 +18,8 @@ "retired": false, "restarting": false, "rebooting": false, - "group": "" + "group": "", + "index": 0 } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java index 8b5cdefddf8..0de065b3eaf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tls; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.hosted.controller.tls.config.TlsConfig; |