diff options
Diffstat (limited to 'controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification')
8 files changed, 349 insertions, 183 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java index 5e36d6d6499..bed053d592f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java @@ -1,6 +1,6 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; -import java.net.URI; import java.util.Objects; /** @@ -9,7 +9,7 @@ import java.util.Objects; * * @author enygaard */ -public record FormattedNotification(Notification notification, String prettyType, String messagePrefix, URI uri) { +public record FormattedNotification(Notification notification, String prettyType, String messagePrefix, String uri) { public FormattedNotification { Objects.requireNonNull(prettyType); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java new file mode 100644 index 00000000000..1c05330702e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java @@ -0,0 +1,101 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.notification; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; +import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; +import com.yahoo.yolean.Exceptions; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; +import org.apache.velocity.app.VelocityEngine; +import org.apache.velocity.runtime.resource.loader.StringResourceLoader; +import org.apache.velocity.runtime.resource.util.StringResourceRepository; +import org.apache.velocity.tools.generic.EscapeTool; + +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; + +/** + * @author bjorncs + */ +public class MailTemplating { + + public enum Template { + MAIL("mail"), DEFAULT_MAIL_CONTENT("default-mail-content"), NOTIFICATION_MESSAGE("notification-message"), + CLOUD_TRIAL_NOTIFICATION("cloud-trial-notification"), MAIL_VERIFICATION("mail-verification"); + + public static Optional<Template> fromId(String id) { + return Arrays.stream(values()).filter(t -> t.id.equals(id)).findAny(); + } + + private final String id; + + Template(String id) { this.id = id; } + + public String getId() { return id; } + } + + private final VelocityEngine velocity; + private final EscapeTool escapeTool = new EscapeTool(); + private final ConsoleUrls consoleUrls; + + public MailTemplating(ConsoleUrls consoleUrls) { + this.velocity = createTemplateEngine(); + this.consoleUrls = consoleUrls; + } + + public String generateDefaultMailHtml(Template mailBodyTemplate, Map<String, Object> params, TenantName tenant) { + var ctx = createVelocityContext(); + ctx.put("accountNotificationLink", consoleUrls.tenantNotifications(tenant)); + ctx.put("privacyPolicyLink", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html"); + ctx.put("termsOfServiceLink", consoleUrls.termsOfService()); + ctx.put("supportLink", consoleUrls.support()); + ctx.put("mailBodyTemplate", mailBodyTemplate.getId()); + params.forEach(ctx::put); + return render(ctx, Template.MAIL); + } + + public String generateMailVerificationHtml(PendingMailVerification pmf) { + var ctx = createVelocityContext(); + ctx.put("verifyLink", consoleUrls.verifyEmail(pmf.getVerificationCode())); + ctx.put("email", pmf.getMailAddress()); + return render(ctx, Template.MAIL_VERIFICATION); + } + + public String escapeHtml(String s) { return escapeTool.html(s); } + + private VelocityContext createVelocityContext() { + var ctx = new VelocityContext(); + ctx.put("esc", escapeTool); + return ctx; + } + + private String render(VelocityContext ctx, Template template) { + var writer = new StringWriter(); + // Ignoring return value - implementation either returns 'true' or throws, never 'false' + velocity.mergeTemplate(template.getId(), StandardCharsets.UTF_8.name(), ctx, writer); + return writer.toString(); + } + + private static VelocityEngine createTemplateEngine() { + var v = new VelocityEngine(); + v.setProperty(Velocity.RESOURCE_LOADERS, "string"); + v.setProperty(Velocity.RESOURCE_LOADER + ".string.class", StringResourceLoader.class.getName()); + v.setProperty(Velocity.RESOURCE_LOADER + ".string.repository.static", "false"); + v.init(); + var repo = (StringResourceRepository) v.getApplicationAttribute(StringResourceLoader.REPOSITORY_NAME_DEFAULT); + Arrays.stream(Template.values()).forEach(t -> registerTemplate(repo, t.getId())); + return v; + } + + private static void registerTemplate(StringResourceRepository repo, String name) { + var templateStr = Exceptions.uncheck(() -> { + var in = MailTemplating.class.getResourceAsStream("/mail/%s.vm".formatted(name)); + return new String(in.readAllBytes()); + }); + repo.putStringResource(name, templateStr); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MissingOptionalException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MissingOptionalException.java index 1379ab4654f..50e4cd40af7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MissingOptionalException.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MissingOptionalException.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java index 53450783c8e..897e0be2d22 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java @@ -1,9 +1,16 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; /** * Represents an event that we want to notify the tenant about. The message(s) should be short @@ -13,15 +20,30 @@ import java.util.Objects; * * @author freva */ -public record Notification(Instant at, com.yahoo.vespa.hosted.controller.notification.Notification.Type type, com.yahoo.vespa.hosted.controller.notification.Notification.Level level, NotificationSource source, List<String> messages) { +public record Notification(Instant at, Notification.Type type, Notification.Level level, NotificationSource source, + String title, List<String> messages, Optional<MailContent> mailContent) { + + public Notification(Instant at, Type type, Level level, NotificationSource source, String title, List<String> messages) { + this(at, type, level, source, title, messages, Optional.empty()); + } public Notification(Instant at, Type type, Level level, NotificationSource source, List<String> messages) { - this.at = Objects.requireNonNull(at, "at cannot be null"); - this.type = Objects.requireNonNull(type, "type cannot be null"); - this.level = Objects.requireNonNull(level, "level cannot be null"); - this.source = Objects.requireNonNull(source, "source cannot be null"); - this.messages = List.copyOf(Objects.requireNonNull(messages, "messages cannot be null")); - if (messages.size() < 1) throw new IllegalArgumentException("messages cannot be empty"); + this(at, type, level, source, "", messages); + } + + public Notification { + Objects.requireNonNull(at, "at cannot be null"); + Objects.requireNonNull(type, "type cannot be null"); + Objects.requireNonNull(level, "level cannot be null"); + Objects.requireNonNull(source, "source cannot be null"); + Objects.requireNonNull(title, "title cannot be null"); + messages = List.copyOf(Objects.requireNonNull(messages, "messages cannot be null")); + + // Allowing empty title temporarily until all notifications have a title + // if (title.isBlank()) throw new IllegalArgumentException("title cannot be empty"); + if (messages.isEmpty() && title.isBlank()) throw new IllegalArgumentException("messages cannot be empty when title is empty"); + + Objects.requireNonNull(mailContent); } public enum Level { @@ -31,36 +53,81 @@ public record Notification(Instant at, com.yahoo.vespa.hosted.controller.notific public enum Type { - /** - * Related to contents of application package, e.g., usage of deprecated features/syntax - */ + /** Related to contents of application package, e.g., usage of deprecated features/syntax */ applicationPackage, - /** - * Related to contents of application package detectable by the controller on submission - */ + /** Related to contents of application package detectable by the controller on submission */ submission, - /** - * Related to contents of application test package, e.g., mismatch between deployment spec and provided tests - */ + /** Related to contents of application test package, e.g., mismatch between deployment spec and provided tests */ testPackage, - /** - * Related to deployment of application, e.g., system test failure, node allocation failure, internal errors, etc. - */ + /** Related to deployment of application, e.g., system test failure, node allocation failure, internal errors, etc. */ deployment, - /** - * Application cluster is (near) external feed blocked - */ + /** Application cluster is (near) external feed blocked */ feedBlock, - /** - * Application cluster is reindexing document(s) - */ - reindex + /** Application cluster is reindexing document(s) */ + reindex, + + /** Account, e.g. expiration of trial plan */ + account, + } + + public static class MailContent { + private final MailTemplating.Template template; + private final SortedMap<String, Object> values; + private final String subject; + + private MailContent(Builder b) { + template = Objects.requireNonNull(b.template); + values = new TreeMap<>(b.values); + subject = b.subject; + } + + public MailTemplating.Template template() { return template; } + public SortedMap<String, Object> values() { return Collections.unmodifiableSortedMap(values); } + public Optional<String> subject() { return Optional.ofNullable(subject); } + + public static Builder fromTemplate(MailTemplating.Template template) { return new Builder(template); } + + public static class Builder { + private final MailTemplating.Template template; + private final Map<String, Object> values = new HashMap<>(); + private String subject; + + private Builder(MailTemplating.Template template) { + this.template = template; + } + + public Builder with(String name, String value) { values.put(name, value); return this; } + public Builder with(String name, Collection<String> items) { values.put(name, List.copyOf(items)); return this; } + public Builder subject(String s) { this.subject = s; return this; } + public MailContent build() { return new MailContent(this); } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MailContent that = (MailContent) o; + return Objects.equals(template, that.template) && Objects.equals(values, that.values) && Objects.equals(subject, that.subject); + } + + @Override + public int hashCode() { + return Objects.hash(template, values, subject); + } + @Override + public String toString() { + return "MailContent{" + + "template='" + template + '\'' + + ", values=" + values + + ", subject='" + subject + '\'' + + '}'; + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java index f753f22608d..e9b38f7a122 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java @@ -1,17 +1,14 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import org.apache.http.client.utils.URIBuilder; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; -import java.net.URI; -import java.net.URISyntaxException; import java.util.Objects; import java.util.Optional; -import java.util.function.Function; + +import static com.yahoo.vespa.hosted.controller.notification.Notifier.notificationLink; /** * Created a NotificationContent for a given Notification. @@ -21,10 +18,10 @@ import java.util.function.Function; * @author enygaard */ public class NotificationFormatter { - private final ZoneRegistry zoneRegistry; + private final ConsoleUrls consoleUrls; - public NotificationFormatter(ZoneRegistry zoneRegistry) { - this.zoneRegistry = Objects.requireNonNull(zoneRegistry); + public NotificationFormatter(ConsoleUrls consoleUrls) { + this.consoleUrls = Objects.requireNonNull(consoleUrls); } public FormattedNotification format(Notification n) { @@ -34,20 +31,18 @@ public class NotificationFormatter { case testPackage -> testPackage(n); case reindex -> reindex(n); case feedBlock -> feedBlock(n); - default -> new FormattedNotification(n, n.type().name(), "", zoneRegistry.dashboardUrl(n.source().tenant())); + default -> new FormattedNotification(n, n.type().name(), "", consoleUrls.tenantOverview(n.source().tenant())); }; } private FormattedNotification applicationPackage(Notification n) { var source = n.source(); var application = requirePresent(source.application(), "application"); - var instance = requirePresent(source.instance(), "instance"); - var message = Text.format("Application package for %s.%s has %s", + var message = Text.format("Application package for %s%s has %s", application, - instance, + source.instance().map(instance -> "." + instance.value()).orElse(""), levelText(n.level(), n.messages().size())); - var uri = zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), application, instance)); - return new FormattedNotification(n, "Application package", message, uri); + return new FormattedNotification(n, "Application package", message, notificationLink(consoleUrls, n.source())); } private FormattedNotification deployment(Notification n) { @@ -57,7 +52,7 @@ public class NotificationFormatter { requirePresent(source.application(), "application"), requirePresent(source.instance(), "instance"), levelText(n.level(), n.messages().size())); - return new FormattedNotification(n,"Deployment", message, jobLink(n.source())); + return new FormattedNotification(n,"Deployment", message, notificationLink(consoleUrls, n.source())); } private FormattedNotification testPackage(Notification n) { @@ -67,68 +62,23 @@ public class NotificationFormatter { n.messages().size() > 1 ? "are problems" : "is a problem", application, source.instance().map(i -> "."+i).orElse("")); - var uri = zoneRegistry.dashboardUrl(source.tenant(), application); - return new FormattedNotification(n, "Test package", message, uri); + return new FormattedNotification(n, "Test package", message, notificationLink(consoleUrls, n.source())); } private FormattedNotification reindex(Notification n) { var message = Text.format("%s is reindexing", clusterInfo(n.source())); - var source = n.source(); - var application = requirePresent(source.application(), "application"); - var instance = requirePresent(source.instance(), "instance"); - var clusterId = requirePresent(source.clusterId(), "clusterId"); - var zone = requirePresent(source.zoneId(), "zoneId"); - var instanceURI = zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), application, instance)); - try { - var uri = new URIBuilder(instanceURI) - .setParameter( - String.format("%s.%s.%s", instance, zone.environment(), zone.region()), - String.format("clusters,%s=status", clusterId.value())) - .build(); - return new FormattedNotification(n, "Reindex", message, uri); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } + var application = requirePresent(n.source().application(), "application"); + var instance = requirePresent(n.source().instance(), "instance"); + var clusterId = requirePresent(n.source().clusterId(), "clusterId"); + var zone = requirePresent(n.source().zoneId(), "zoneId"); + return new FormattedNotification(n, "Reindex", message, + consoleUrls.clusterReindexing(ApplicationId.from(n.source().tenant(), application, instance), zone, clusterId)); } private FormattedNotification feedBlock(Notification n) { - String type; - if (n.level() == Notification.Level.warning) { - type = "Nearly feed blocked"; - } else { - type = "Feed blocked"; - } + String type = n.level() == Notification.Level.warning ? "Nearly feed blocked" : "Feed blocked"; var message = Text.format("%s is %s", clusterInfo(n.source()), type.toLowerCase()); - var source = n.source(); - var application = requirePresent(source.application(), "application"); - var instance = requirePresent(source.instance(), "instance"); - var clusterId = requirePresent(source.clusterId(), "clusterId"); - var zone = requirePresent(source.zoneId(), "zoneId"); - var instanceURI = zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), application, instance)); - try { - var uri = new URIBuilder(instanceURI) - .setParameter( - String.format("%s.%s.%s", instance, zone.environment(), zone.region()), - String.format("clusters,%s", clusterId.value())) - .build(); - return new FormattedNotification(n, type, message, uri); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } - } - - private URI jobLink(NotificationSource source) { - var application = requirePresent(source.application(), "application"); - var instance = requirePresent(source.instance(), "instance"); - var jobType = requirePresent(source.jobType(), "jobType"); - var runNumber = source.runNumber().orElseThrow(() -> new MissingOptionalException("runNumber")); - var applicationId = ApplicationId.from(source.tenant(), application, instance); - Function<Environment, URI> link = (Environment env) -> zoneRegistry.dashboardUrl(new RunId(applicationId, jobType, runNumber)); - var environment = jobType.zone().environment(); - return switch (environment) { - case dev, perf -> link.apply(environment); - default -> link.apply(Environment.prod); - }; + return new FormattedNotification(n, type, message, notificationLink(consoleUrls, n.source())); } private String jobText(NotificationSource source) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java index c414e24a187..72d3dd933aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index f8505775d26..e279e4feacd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; import com.yahoo.collections.Pair; @@ -9,7 +9,11 @@ import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.notification.Notification.MailContent; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Clock; @@ -18,12 +22,14 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing.Cluster; import static com.yahoo.vespa.hosted.controller.notification.Notification.Level; import static com.yahoo.vespa.hosted.controller.notification.Notification.Type; +import static com.yahoo.vespa.hosted.controller.notification.Notifier.notificationLink; /** * Adds, updates and removes tenant notifications in ZK @@ -32,18 +38,22 @@ import static com.yahoo.vespa.hosted.controller.notification.Notification.Type; */ public class NotificationsDb { + private static final Logger log = Logger.getLogger(NotificationsDb.class.getName()); + private final Clock clock; private final CuratorDb curatorDb; private final Notifier notifier; + private final ConsoleUrls consoleUrls; public NotificationsDb(Controller controller) { - this(controller.clock(), controller.curator(), controller.notifier()); + this(controller.clock(), controller.curator(), controller.notifier(), controller.serviceRegistry().consoleUrls()); } - NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier) { + NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier, ConsoleUrls consoleUrls) { this.clock = clock; this.curatorDb = curatorDb; this.notifier = notifier; + this.consoleUrls = consoleUrls; } public List<TenantName> listTenantsWithNotifications() { @@ -56,29 +66,69 @@ public class NotificationsDb { .toList(); } - public void setNotification(NotificationSource source, Type type, Level level, String message) { - setNotification(source, type, level, List.of(message)); + public void setSubmissionNotification(TenantAndApplicationId tenantApp, String message) { + NotificationSource source = NotificationSource.from(tenantApp); + String title = "Application package for [%s](%s) has a warning".formatted( + tenantApp.application().value(), notificationLink(consoleUrls, source)); + setNotification(source, Type.submission, Level.warning, title, List.of(message), Optional.empty()); + } + + public void setApplicationPackageNotification(NotificationSource source, List<String> messages) { + String title = "Application package for [%s%s](%s) has %s".formatted( + source.application().get().value(), source.instance().map(i -> "." + i.value()).orElse(""), notificationLink(consoleUrls, source), + messages.size() == 1 ? "a warning" : "warnings"); + setNotification(source, Type.applicationPackage, Level.warning, title, messages, Optional.empty()); + } + + public void setTestPackageNotification(TenantAndApplicationId tenantApp, List<String> messages) { + NotificationSource source = NotificationSource.from(tenantApp); + String title = "There %s with tests for [%s](%s)".formatted( + messages.size() == 1 ? "is a problem" : "are problems", tenantApp.application().value(), + notificationLink(consoleUrls, source)); + setNotification(source, Type.testPackage, Level.warning, title, messages, Optional.empty()); + } + + public void setDeploymentNotification(RunId runId, String message) { + String description, linkText; + if (runId.type().isProduction()) { + description = runId.type().isTest() ? "Test job " : "Deployment job "; + linkText = "#" + runId.number() + " to " + runId.type().zone().region().value(); + } else if (runId.type().isTest()) { + description = ""; + linkText = (runId.type().isStagingTest() ? "Staging" : "System") + " test #" + runId.number(); + } else if (runId.type().isDeployment()) { + description = "Deployment job "; + linkText = "#" + runId.number() + " to " + runId.type().zone().value(); + } else throw new IllegalStateException("Unexpected job type " + runId.type()); + NotificationSource source = NotificationSource.from(runId); + String title = "%s[%s](%s) for application **%s.%s** has failed".formatted( + description, linkText, notificationLink(consoleUrls, source), runId.application().application().value(), runId.application().instance().value()); + setNotification(source, Type.deployment, Level.error, title, List.of(message), Optional.empty()); } /** * Add a notification with given source and type. If a notification with same source and type - * already exists, it'll be replaced by this one instead + * already exists, it'll be replaced by this one instead. */ - public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) { + public void setNotification(NotificationSource source, Type type, Level level, String title, List<String> messages, + Optional<MailContent> mailContent) { Optional<Notification> changed = Optional.empty(); try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { var existingNotifications = curatorDb.readNotifications(source.tenant()); List<Notification> notifications = existingNotifications.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) .collect(Collectors.toCollection(ArrayList::new)); - var notification = new Notification(clock.instant(), type, level, source, messages); + var notification = new Notification(clock.instant(), type, level, source, title, messages, mailContent); if (!notificationExists(notification, existingNotifications, false)) { changed = Optional.of(notification); } notifications.add(notification); curatorDb.writeNotifications(source.tenant(), notifications); } - changed.ifPresent(notifier::dispatch); + changed.ifPresent(c -> { + log.fine(() -> "New notification %s".formatted(c)); + notifier.dispatch(c); + }); } /** Remove the notification with the given source and type */ @@ -122,14 +172,9 @@ public class NotificationsDb { Instant now = clock.instant(); List<Notification> changed = List.of(); List<Notification> newNotifications = Stream.concat( - clusterMetrics.stream().map(metric -> { - NotificationSource source = NotificationSource.from(deploymentId, ClusterSpec.Id.from(metric.getClusterId())); - return createFeedBlockNotification(source, now, metric); - }), - applicationReindexing.clusters().entrySet().stream().map(entry -> { - NotificationSource source = NotificationSource.from(deploymentId, ClusterSpec.Id.from(entry.getKey())); - return createReindexNotification(source, now, entry.getValue()); - })) + clusterMetrics.stream().map(metric -> createFeedBlockNotification(consoleUrls, deploymentId, metric.getClusterId(), now, metric)), + applicationReindexing.clusters().entrySet().stream().map(entry -> + createReindexNotification(consoleUrls, deploymentId, entry.getKey(), now, entry.getValue()))) .flatMap(Optional::stream) .toList(); @@ -156,30 +201,41 @@ public class NotificationsDb { private boolean notificationExists(Notification notification, List<Notification> existing, boolean mindHigherLevel) { // Be conservative for now, only dispatch notifications if they are from new source or with new type. // the message content and level is ignored for now - return existing.stream().anyMatch(e -> - notification.source().contains(e.source()) && notification.type().equals(e.type()) && + boolean exists = existing.stream() + .anyMatch(e -> notification.source().contains(e.source()) && notification.type().equals(e.type()) && (!mindHigherLevel || notification.level().ordinal() <= e.level().ordinal())); + log.fine(() -> "%s in %s == %b".formatted(notification, existing, exists)); + return exists; } - private static Optional<Notification> createFeedBlockNotification(NotificationSource source, Instant at, ClusterMetrics metric) { + private static Optional<Notification> createFeedBlockNotification(ConsoleUrls consoleUrls, DeploymentId deployment, String clusterId, Instant at, ClusterMetrics metric) { Optional<Pair<Level, String>> memoryStatus = resourceUtilToFeedBlockStatus("memory", metric.memoryUtil(), metric.memoryFeedBlockLimit()); Optional<Pair<Level, String>> diskStatus = resourceUtilToFeedBlockStatus("disk", metric.diskUtil(), metric.diskFeedBlockLimit()); if (memoryStatus.isEmpty() && diskStatus.isEmpty()) return Optional.empty(); + NotificationSource source = NotificationSource.from(deployment, ClusterSpec.Id.from(clusterId)); // Find the max among levels Level level = Stream.of(memoryStatus, diskStatus) .flatMap(status -> status.stream().map(Pair::getFirst)) .max(Comparator.comparing(Enum::ordinal)).get(); + String title = "Cluster [%s](%s) in **%s** for **%s.%s** is %sfeed blocked".formatted( + clusterId, notificationLink(consoleUrls, source), deployment.zoneId().value(), deployment.applicationId().application().value(), + deployment.applicationId().instance().value(), level == Level.warning ? "nearly " : ""); List<String> messages = Stream.concat(memoryStatus.stream(), diskStatus.stream()) .filter(status -> status.getFirst() == level) // Do not mix message from different levels .map(Pair::getSecond) .toList(); - return Optional.of(new Notification(at, Type.feedBlock, level, source, messages)); + + return Optional.of(new Notification(at, Type.feedBlock, level, source, title, messages)); } - private static Optional<Notification> createReindexNotification(NotificationSource source, Instant at, Cluster cluster) { + private static Optional<Notification> createReindexNotification(ConsoleUrls consoleUrls, DeploymentId deployment, String clusterId, Instant at, Cluster cluster) { + NotificationSource source = NotificationSource.from(deployment, ClusterSpec.Id.from(clusterId)); + String title = "Cluster [%s](%s) in **%s** for **%s.%s** is [reindexing](https://docs.vespa.ai/en/operations/reindexing.html)".formatted( + clusterId, consoleUrls.clusterReindexing(deployment.applicationId(), deployment.zoneId(), source.clusterId().get()), + deployment.zoneId().value(), deployment.applicationId().application().value(), deployment.applicationId().instance().value()); List<String> messages = cluster.ready().entrySet().stream() .filter(entry -> entry.getValue().progress().isPresent()) .map(entry -> Text.format("document type '%s'%s (%.1f%% done)", @@ -187,7 +243,7 @@ public class NotificationsDb { .sorted() .toList(); if (messages.isEmpty()) return Optional.empty(); - return Optional.of(new Notification(at, Type.reindex, Level.info, source, messages)); + return Optional.of(new Notification(at, Type.reindex, Level.info, source, title, messages)); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java index 82dc333d178..f27e69c4636 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java @@ -1,24 +1,23 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; import com.google.common.annotations.VisibleForTesting; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.TenantName; -import com.yahoo.restapi.UriBuilder; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; -import java.net.URI; import java.util.Collection; import java.util.List; import java.util.Objects; @@ -27,8 +26,6 @@ import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static com.yahoo.yolean.Exceptions.uncheck; - /** * Notifier is responsible for dispatching user notifications to their chosen Contact points. * @@ -38,20 +35,22 @@ public class Notifier { private final CuratorDb curatorDb; private final Mailer mailer; private final FlagSource flagSource; + private final ConsoleUrls consoleUrls; private final NotificationFormatter formatter; - private final URI dashboardUri; + private final MailTemplating mailTemplating; private static final Logger log = Logger.getLogger(Notifier.class.getName()); // Minimal url pattern matcher to detect hardcoded URLs in Notification messages private static final Pattern urlPattern = Pattern.compile("https://[\\w\\d./]+"); - public Notifier(CuratorDb curatorDb, ZoneRegistry zoneRegistry, Mailer mailer, FlagSource flagSource) { + public Notifier(CuratorDb curatorDb, ConsoleUrls consoleUrls, Mailer mailer, FlagSource flagSource) { this.curatorDb = Objects.requireNonNull(curatorDb); this.mailer = Objects.requireNonNull(mailer); this.flagSource = Objects.requireNonNull(flagSource); - this.formatter = new NotificationFormatter(zoneRegistry); - this.dashboardUri = zoneRegistry.dashboardUrl(); + this.consoleUrls = Objects.requireNonNull(consoleUrls); + this.formatter = new NotificationFormatter(consoleUrls); + this.mailTemplating = new MailTemplating(consoleUrls); } public void dispatch(List<Notification> notifications, NotificationSource source) { @@ -99,11 +98,16 @@ public class Notifier { private void dispatch(Notification notification, Collection<TenantContacts.EmailContact> contacts) { try { + log.fine(() -> "Sending notification " + notification + " to " + + contacts.stream().map(c -> c.email().getEmailAddress()).toList()); var content = formatter.format(notification); - mailer.send(mailOf(content, contacts.stream() - .filter(c -> c.email().isVerified()) - .map(c -> c.email().getEmailAddress()) - .toList())); + var verifiedContacts = contacts.stream() + .filter(c -> c.email().isVerified()).map(c -> c.email().getEmailAddress()).toList(); + if (verifiedContacts.isEmpty()) { + log.fine(() -> "None of the %d contact(s) are verified - skipping delivery of %s".formatted(contacts.size(), notification)); + return; + } + mailer.send(mailOf(content, verifiedContacts)); } catch (MailerException e) { log.log(Level.SEVERE, "Failed sending email", e); } catch (MissingOptionalException e) { @@ -113,23 +117,30 @@ public class Notifier { public Mail mailOf(FormattedNotification content, Collection<String> recipients) { var notification = content.notification(); - var subject = Text.format("[%s] %s Vespa Notification for %s", notification.level().toString().toUpperCase(), content.prettyType(), applicationIdSource(notification.source())); - var template = uncheck(() -> Notifier.class.getResourceAsStream("/mail/mail-notification.tmpl").readAllBytes()); - var html = new String(template) - .replace("[[NOTIFICATION_HEADER]]", content.messagePrefix()) - .replace("[[NOTIFICATION_ITEMS]]", notification.messages().stream() - .map(Notifier::linkify) - .map(Notifier::capitalise) - .map(m -> "<p>" + m + "</p>") - .collect(Collectors.joining())) - .replace("[[LINK_TO_NOTIFICATION]]", notificationLink(notification.source())) - .replace("[[LINK_TO_ACCOUNT_NOTIFICATIONS]]", accountNotificationsUri(content.notification().source().tenant())) - .replace("[[LINK_TO_PRIVACY_POLICY]]", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html") - .replace("[[LINK_TO_TERMS_OF_SERVICE]]", consoleUri("terms-of-service-trial.html")) - .replace("[[LINK_TO_SUPPORT]]", consoleUri("support")); + var subject = content.notification().mailContent().flatMap(Notification.MailContent::subject) + .orElseGet(() -> Text.format( + "[%s] %s Vespa Notification for %s", notification.level().toString().toUpperCase(), + content.prettyType(), applicationIdSource(notification.source()))); + var html = generateHtml(content); return new Mail(recipients, subject, "", html); } + private String generateHtml(FormattedNotification content) { + var mailContent = content.notification().mailContent().orElseGet(() -> generateContentFromMessages(content)); + return mailTemplating.generateDefaultMailHtml(mailContent.template(), mailContent.values(), content.notification().source().tenant()); + } + + private Notification.MailContent generateContentFromMessages(FormattedNotification f) { + var items = f.notification().messages().stream().map(m -> capitalise(linkify(mailTemplating.escapeHtml(m)))).toList(); + return Notification.MailContent.fromTemplate(MailTemplating.Template.DEFAULT_MAIL_CONTENT) + .with("mailMessageTemplate", "notification-message") + .with("mailTitle", "Vespa Cloud Notifications") + .with("notificationHeader", f.messagePrefix()) + .with("notificationItems", items) + .with("consoleLink", notificationLink(consoleUrls, f.notification().source())) + .build(); + } + @VisibleForTesting static String linkify(String text) { return urlPattern.matcher(text).replaceAll((res) -> String.format("<a href=\"%s\">%s</a>", res.group(), res.group())); @@ -143,36 +154,16 @@ public class Notifier { return sb.toString(); } - private String accountNotificationsUri(TenantName tenant) { - return new UriBuilder(dashboardUri) - .append("tenant/") - .append(tenant.value()) - .append("account/notifications") - .toString(); - } + static String notificationLink(ConsoleUrls consoleUrls, NotificationSource source) { + if (source.application().isEmpty()) return consoleUrls.tenantOverview(source.tenant()); + if (source.instance().isEmpty()) return consoleUrls.prodApplicationOverview(source.tenant(), source.application().get()); - private String consoleUri(String path) { - return new UriBuilder(dashboardUri).append(path).toString(); - } - - private String notificationLink(NotificationSource source) { - var uri = new UriBuilder(dashboardUri); - uri = uri.append("tenant").append(source.tenant().value()); - if (source.application().isPresent()) - uri = uri.append("application").append(source.application().get().value()); - if (source.isProduction()) { - uri = uri.append("prod/instance"); - if (source.jobType().isPresent()) { - uri = uri.append(source.instance().get().value()); - } - } - else { - uri = uri.append("dev/instance/").append(source.instance().get().value()); - } - if (source.jobType().isPresent()) { - uri = uri.append("job").append(source.jobType().get().jobName()).append("run").append(String.valueOf(source.runNumber().getAsLong())); - } - return uri.toString(); + ApplicationId application = ApplicationId.from(source.tenant(), source.application().get(), source.instance().get()); + if (source.jobType().isPresent()) + return consoleUrls.deploymentRun(new RunId(application, source.jobType().get(), source.runNumber().getAsLong())); + if (source.clusterId().isPresent()) + return consoleUrls.clusterOverview(application, source.zoneId().get(), source.clusterId().get()); + return consoleUrls.instanceOverview(application, source.zoneId().map(ZoneId::environment).orElse(Environment.prod)); } private static String capitalise(String m) { |