Set title for existing notifications

11 files changed, 151 insertions, 90 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 0de0ea06904..d7a3d4fb9e5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -554,7 +554,7 @@ public class ApplicationController {
                 if (warnings.isEmpty())
                     controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage);
                 else
-                    controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings);
+                    controller.notificationsDb().setApplicationPackageNotification(source, warnings);
             }
 
             lockApplicationOrThrow(applicationId, application ->
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index d62e477a51f..9bfa2674754 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -856,7 +856,7 @@ public class InternalStepRunner implements StepRunner {
 
     private void updateConsoleNotification(Run run, boolean isRemoved) {
         NotificationSource source = NotificationSource.from(run.id());
-        Consumer<String> updater = msg -> controller.notificationsDb().setNotification(source, Notification.Type.deployment, Notification.Level.error, msg);
+        Consumer<String> updater = msg -> controller.notificationsDb().setDeploymentNotification(run.id(), msg);
         switch (isRemoved ? success : run.status()) {
             case aborted, cancelled: return; // wait and see how the next run goes.
             case noTests:
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 0dc30f54d61..ae6bcdea00c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -37,7 +37,6 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff;
 import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage;
 import com.yahoo.vespa.hosted.controller.deployment.Run.Reason;
-import com.yahoo.vespa.hosted.controller.notification.Notification;
 import com.yahoo.vespa.hosted.controller.notification.Notification.Type;
 import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
@@ -625,19 +624,15 @@ public class JobController {
     private void validateTests(TenantAndApplicationId id, Submission submission) {
         var testSummary = TestPackage.validateTests(submission.applicationPackage().deploymentSpec(), submission.testPackage());
         if ( ! testSummary.problems().isEmpty())
-            controller.notificationsDb().setNotification(NotificationSource.from(id),
-                                                         Type.testPackage,
-                                                         Notification.Level.warning,
-                                                         testSummary.problems());
-
+            controller.notificationsDb().setTestPackageNotification(id, testSummary.problems());
     }
 
     private void validateMajorVersion(TenantAndApplicationId id, Submission submission) {
         submission.applicationPackage().deploymentSpec().majorVersion().ifPresent(explicitMajor -> {
             if ( ! controller.readVersionStatus().isOnCurrentMajor(new Version(explicitMajor)))
-                controller.notificationsDb().setNotification(NotificationSource.from(id), Type.submission, Notification.Level.warning,
-                                                             "Vespa " + explicitMajor + " will soon reach end of life, upgrade to Vespa " + (explicitMajor + 1) + " now: " +
-                                                             "https://cloud.vespa.ai/en/vespa" + (explicitMajor + 1) + "-release-notes.html"); // ∠( ᐛ 」∠)＿
+                controller.notificationsDb().setSubmissionNotification(id,
+                                                             "Vespa " + explicitMajor + " will soon reach end of life, upgrade to [Vespa " + (explicitMajor + 1) + " now](" +
+                                                             "https://cloud.vespa.ai/en/vespa" + (explicitMajor + 1) + "-release-notes.html)"); // ∠( ᐛ 」∠)＿
         });
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
index 4a94098ce98..f28b21228d9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
@@ -32,18 +32,18 @@ public record Notification(Instant at, Notification.Type type, Notification.Leve
     }
 
     public Notification {
-        at = Objects.requireNonNull(at, "at cannot be null");
-        type = Objects.requireNonNull(type, "type cannot be null");
-        level = Objects.requireNonNull(level, "level cannot be null");
-        source = Objects.requireNonNull(source, "source cannot be null");
-        title = Objects.requireNonNull(title, "title cannot be null");
-        messages = List.copyOf(Objects.requireNonNull(messages, "messages cannot be null"));
+        Objects.requireNonNull(at, "at cannot be null");
+        Objects.requireNonNull(type, "type cannot be null");
+        Objects.requireNonNull(level, "level cannot be null");
+        Objects.requireNonNull(source, "source cannot be null");
+        Objects.requireNonNull(title, "title cannot be null");
+        List.copyOf(Objects.requireNonNull(messages, "messages cannot be null"));
 
         // Allowing empty title temporarily until all notifications have a title
         // if (title.isBlank()) throw new IllegalArgumentException("title cannot be empty");
         if (messages.isEmpty() && title.isBlank()) throw new IllegalArgumentException("messages cannot be empty when title is empty");
 
-        mailContent = Objects.requireNonNull(mailContent);
+        Objects.requireNonNull(mailContent);
     }
 
     public enum Level {
@@ -53,40 +53,26 @@ public record Notification(Instant at, Notification.Type type, Notification.Leve
 
     public enum Type {
 
-        /**
-         * Related to contents of application package, e.g., usage of deprecated features/syntax
-         */
+        /** Related to contents of application package, e.g., usage of deprecated features/syntax */
         applicationPackage,
 
-        /**
-         * Related to contents of application package detectable by the controller on submission
-         */
+        /** Related to contents of application package detectable by the controller on submission */
         submission,
 
-        /**
-         * Related to contents of application test package, e.g., mismatch between deployment spec and provided tests
-         */
+        /** Related to contents of application test package, e.g., mismatch between deployment spec and provided tests */
         testPackage,
 
-        /**
-         * Related to deployment of application, e.g., system test failure, node allocation failure, internal errors, etc.
-         */
+        /** Related to deployment of application, e.g., system test failure, node allocation failure, internal errors, etc. */
         deployment,
 
-        /**
-         * Application cluster is (near) external feed blocked
-         */
+        /** Application cluster is (near) external feed blocked */
         feedBlock,
 
-        /**
-         * Application cluster is reindexing document(s)
-         */
+        /** Application cluster is reindexing document(s) */
         reindex,
 
-        /**
-         * Account, e.g. expiration of trial plan
-         */
-        account
+        /** Account, e.g. expiration of trial plan */
+        account,
     }
 
     public static class MailContent {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java
index 243e1af8f35..caf25a7baea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java
@@ -38,10 +38,9 @@ public class NotificationFormatter {
     private FormattedNotification applicationPackage(Notification n) {
         var source = n.source();
         var application = requirePresent(source.application(), "application");
-        var instance = requirePresent(source.instance(), "instance");
-        var message = Text.format("Application package for %s.%s has %s",
+        var message = Text.format("Application package for %s%s has %s",
                 application,
-                instance,
+                source.instance().map(instance -> "." + instance.value()),
                 levelText(n.level(), n.messages().size()));
         return new FormattedNotification(n, "Application package", message, notificationLink(consoleUrls, n.source()));
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java
index 081fd5a2c1d..e279e4feacd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java
@@ -9,7 +9,10 @@ import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.notification.Notification.MailContent;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
@@ -26,6 +29,7 @@ import java.util.stream.Stream;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing.Cluster;
 import static com.yahoo.vespa.hosted.controller.notification.Notification.Level;
 import static com.yahoo.vespa.hosted.controller.notification.Notification.Type;
+import static com.yahoo.vespa.hosted.controller.notification.Notifier.notificationLink;
 
 /**
  * Adds, updates and removes tenant notifications in ZK
@@ -39,15 +43,17 @@ public class NotificationsDb {
     private final Clock clock;
     private final CuratorDb curatorDb;
     private final Notifier notifier;
+    private final ConsoleUrls consoleUrls;
 
     public NotificationsDb(Controller controller) {
-        this(controller.clock(), controller.curator(), controller.notifier());
+        this(controller.clock(), controller.curator(), controller.notifier(), controller.serviceRegistry().consoleUrls());
     }
 
-    NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier) {
+    NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier, ConsoleUrls consoleUrls) {
         this.clock = clock;
         this.curatorDb = curatorDb;
         this.notifier = notifier;
+        this.consoleUrls = consoleUrls;
     }
 
     public List<TenantName> listTenantsWithNotifications() {
@@ -60,12 +66,44 @@ public class NotificationsDb {
                 .toList();
     }
 
-    public void setNotification(NotificationSource source, Type type, Level level, String message) {
-        setNotification(source, type, level, List.of(message));
+    public void setSubmissionNotification(TenantAndApplicationId tenantApp, String message) {
+        NotificationSource source = NotificationSource.from(tenantApp);
+        String title = "Application package for [%s](%s) has a warning".formatted(
+                tenantApp.application().value(), notificationLink(consoleUrls, source));
+        setNotification(source, Type.submission, Level.warning, title, List.of(message), Optional.empty());
     }
 
-    public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) {
-        setNotification(source, type, level, "", messages, Optional.empty());
+    public void setApplicationPackageNotification(NotificationSource source, List<String> messages) {
+        String title = "Application package for [%s%s](%s) has %s".formatted(
+                source.application().get().value(), source.instance().map(i -> "." + i.value()).orElse(""), notificationLink(consoleUrls, source),
+                messages.size() == 1 ? "a warning" : "warnings");
+        setNotification(source, Type.applicationPackage, Level.warning, title, messages, Optional.empty());
+    }
+
+    public void setTestPackageNotification(TenantAndApplicationId tenantApp, List<String> messages) {
+        NotificationSource source = NotificationSource.from(tenantApp);
+        String title = "There %s with tests for [%s](%s)".formatted(
+                messages.size() == 1 ? "is a problem" : "are problems", tenantApp.application().value(),
+                notificationLink(consoleUrls, source));
+        setNotification(source, Type.testPackage, Level.warning, title, messages, Optional.empty());
+    }
+
+    public void setDeploymentNotification(RunId runId, String message) {
+        String description, linkText;
+        if (runId.type().isProduction()) {
+            description = runId.type().isTest() ? "Test job " : "Deployment job ";
+            linkText = "#" + runId.number() + " to " + runId.type().zone().region().value();
+        } else if (runId.type().isTest()) {
+            description = "";
+            linkText = (runId.type().isStagingTest() ? "Staging" : "System") + " test #" + runId.number();
+        } else if (runId.type().isDeployment()) {
+            description = "Deployment job ";
+            linkText = "#" + runId.number() + " to " + runId.type().zone().value();
+        } else throw new IllegalStateException("Unexpected job type " + runId.type());
+        NotificationSource source = NotificationSource.from(runId);
+        String title = "%s[%s](%s) for application **%s.%s** has failed".formatted(
+                description, linkText, notificationLink(consoleUrls, source), runId.application().application().value(), runId.application().instance().value());
+        setNotification(source, Type.deployment, Level.error, title, List.of(message), Optional.empty());
     }
 
     /**
@@ -134,14 +172,9 @@ public class NotificationsDb {
         Instant now = clock.instant();
         List<Notification> changed = List.of();
         List<Notification> newNotifications = Stream.concat(
-                clusterMetrics.stream().map(metric -> {
-                    NotificationSource source = NotificationSource.from(deploymentId, ClusterSpec.Id.from(metric.getClusterId()));
-                    return createFeedBlockNotification(source, now, metric);
-                }),
-                applicationReindexing.clusters().entrySet().stream().map(entry -> {
-                    NotificationSource source = NotificationSource.from(deploymentId, ClusterSpec.Id.from(entry.getKey()));
-                    return createReindexNotification(source, now, entry.getValue());
-                }))
+                clusterMetrics.stream().map(metric -> createFeedBlockNotification(consoleUrls, deploymentId, metric.getClusterId(), now, metric)),
+                applicationReindexing.clusters().entrySet().stream().map(entry ->
+                        createReindexNotification(consoleUrls, deploymentId, entry.getKey(), now, entry.getValue())))
                 .flatMap(Optional::stream)
                 .toList();
 
@@ -175,25 +208,34 @@ public class NotificationsDb {
         return exists;
     }
 
-    private static Optional<Notification> createFeedBlockNotification(NotificationSource source, Instant at, ClusterMetrics metric) {
+    private static Optional<Notification> createFeedBlockNotification(ConsoleUrls consoleUrls, DeploymentId deployment, String clusterId, Instant at, ClusterMetrics metric) {
         Optional<Pair<Level, String>> memoryStatus =
                 resourceUtilToFeedBlockStatus("memory", metric.memoryUtil(), metric.memoryFeedBlockLimit());
         Optional<Pair<Level, String>> diskStatus =
                 resourceUtilToFeedBlockStatus("disk", metric.diskUtil(), metric.diskFeedBlockLimit());
         if (memoryStatus.isEmpty() && diskStatus.isEmpty()) return Optional.empty();
 
+        NotificationSource source = NotificationSource.from(deployment, ClusterSpec.Id.from(clusterId));
         // Find the max among levels
         Level level = Stream.of(memoryStatus, diskStatus)
                 .flatMap(status -> status.stream().map(Pair::getFirst))
                 .max(Comparator.comparing(Enum::ordinal)).get();
+        String title = "Cluster [%s](%s) in **%s** for **%s.%s** is %sfeed blocked".formatted(
+                clusterId, notificationLink(consoleUrls, source), deployment.zoneId().value(), deployment.applicationId().application().value(),
+                deployment.applicationId().instance().value(), level == Level.warning ? "nearly " : "");
         List<String> messages = Stream.concat(memoryStatus.stream(), diskStatus.stream())
                 .filter(status -> status.getFirst() == level) // Do not mix message from different levels
                 .map(Pair::getSecond)
                 .toList();
-        return Optional.of(new Notification(at, Type.feedBlock, level, source, "", messages));
+
+        return Optional.of(new Notification(at, Type.feedBlock, level, source, title, messages));
     }
 
-    private static Optional<Notification> createReindexNotification(NotificationSource source, Instant at, Cluster cluster) {
+    private static Optional<Notification> createReindexNotification(ConsoleUrls consoleUrls, DeploymentId deployment, String clusterId, Instant at, Cluster cluster) {
+        NotificationSource source = NotificationSource.from(deployment, ClusterSpec.Id.from(clusterId));
+        String title = "Cluster [%s](%s) in **%s** for **%s.%s** is [reindexing](https://docs.vespa.ai/en/operations/reindexing.html)".formatted(
+                clusterId, consoleUrls.clusterReindexing(deployment.applicationId(), deployment.zoneId(), source.clusterId().get()),
+                deployment.zoneId().value(), deployment.applicationId().application().value(), deployment.applicationId().instance().value());
         List<String> messages = cluster.ready().entrySet().stream()
                 .filter(entry -> entry.getValue().progress().isPresent())
                 .map(entry -> Text.format("document type '%s'%s (%.1f%% done)",
@@ -201,7 +243,7 @@ public class NotificationsDb {
                 .sorted()
                 .toList();
         if (messages.isEmpty()) return Optional.empty();
-        return Optional.of(new Notification(at, Type.reindex, Level.info, source, "", messages));
+        return Optional.of(new Notification(at, Type.reindex, Level.info, source, title, messages));
     }
 
     /**
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index eb86f23fbfb..345c880eaea 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -1345,6 +1345,7 @@ public class ControllerTest {
                         Type.testPackage,
                         Level.warning,
                         NotificationSource.from(app.application().id()),
+                        "There are problems with tests for [application](https://console.tld/tenant/tenant/application/application/prod/instance)",
                         List.of("test package has staging tests, so it should also include staging setup",
                                 "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"))),
                 tester.controller().notificationsDb().listNotifications(NotificationSource.from(app.application().id()), true));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
index 39c3f0f2b74..e41be11c846 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.notification;
 
 import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
@@ -17,10 +18,12 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
+import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess;
 import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
@@ -85,7 +88,8 @@ public class NotificationsDbTest {
     private final MockCuratorDb curatorDb = new MockCuratorDb(SystemName.Public);
     private final MockMailer mailer = new MockMailer();
     private final FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(PermanentFlags.NOTIFICATION_DISPATCH_FLAG.id(), true);
-    private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, new ConsoleUrls(URI.create("https://console.tld")), mailer, flagSource));
+    private final ConsoleUrls consoleUrls = new ConsoleUrls(URI.create("https://console.tld"));
+    private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, consoleUrls, mailer, flagSource), consoleUrls);
 
     @Test
     void list_test() {
@@ -103,10 +107,10 @@ public class NotificationsDbTest {
         Notification notification2 = notification(12345, Type.deployment, Level.error,   NotificationSource.from(ApplicationId.from(tenant.value(), "app3", "instance2")), "instance msg #3");
 
         // Replace the 3rd notification
-        notificationsDb.setNotification(notification1.source(), notification1.type(), notification1.level(), notification1.messages());
+        setNotification(notification1);
 
         // Notification for a new app, add without replacement
-        notificationsDb.setNotification(notification2.source(), notification2.type(), notification2.level(), notification2.messages());
+        setNotification(notification2);
 
         List<Notification> expected = notificationIndices(0, 1, 3, 4, 5);
         expected.addAll(List.of(notification1, notification2));
@@ -120,19 +124,19 @@ public class NotificationsDbTest {
         Notification notification3 = notification(12345, Type.reindex, Level.warning, NotificationSource.from(new DeploymentId(ApplicationId.from(tenant.value(), "app2", "instance2"), ZoneId.defaultId()), new ClusterSpec.Id("content")), "instance msg #2");
         ;
         var a = notifications.get(0);
-        notificationsDb.setNotification(a.source(), a.type(), a.level(), a.messages());
+        setNotification(a);
         assertEquals(0, mailer.inbox(email.getEmailAddress()).size());
 
         // Replace the 3rd notification. but don't change source or type
-        notificationsDb.setNotification(notification1.source(), notification1.type(), notification1.level(), notification1.messages());
+        setNotification(notification1);
         assertEquals(0, mailer.inbox(email.getEmailAddress()).size());
 
         // Notification for a new app, add without replacement
-        notificationsDb.setNotification(notification2.source(), notification2.type(), notification2.level(), notification2.messages());
+        setNotification(notification2);
         assertEquals(1, mailer.inbox(email.getEmailAddress()).size());
 
         // Notification for new type on existing app
-        notificationsDb.setNotification(notification3.source(), notification3.type(), notification3.level(), notification3.messages());
+        setNotification(notification3);
         assertEquals(2, mailer.inbox(email.getEmailAddress()).size());
     }
 
@@ -200,17 +204,19 @@ public class NotificationsDbTest {
 
         // One resource is at warning
         notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.88, 0.9, 0.3, 0.5)), emptyReindexing);
-        expected.add(notification(12345, Type.feedBlock, Level.warning, sourceCluster1, "disk (usage: 88.0%, feed block limit: 90.0%)"));
+        expected.add(notification(12345, Type.feedBlock, Level.warning, "Cluster [cluster1](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster1) in **prod.us-south-3** for **app1.instance1** is nearly feed blocked",
+                sourceCluster1, "disk (usage: 88.0%, feed block limit: 90.0%)"));
         assertEquals(expected, curatorDb.readNotifications(tenant));
 
         // Both resources over the limit
         notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, 0.9, 0.3, 0.5)), emptyReindexing);
-        expected.set(6, notification(12345, Type.feedBlock, Level.error, sourceCluster1, "disk (usage: 95.0%, feed block limit: 90.0%)"));
+        expected.set(6, notification(12345, Type.feedBlock, Level.error, "Cluster [cluster1](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster1) in **prod.us-south-3** for **app1.instance1** is feed blocked",
+                sourceCluster1, "disk (usage: 95.0%, feed block limit: 90.0%)"));
         assertEquals(expected, curatorDb.readNotifications(tenant));
 
         // One resource at warning, one at error: Only show error message
         notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, 0.9, 0.7, 0.5)), emptyReindexing);
-        expected.set(6, notification(12345, Type.feedBlock, Level.error, sourceCluster1,
+        expected.set(6, notification(12345, Type.feedBlock, Level.error, "Cluster [cluster1](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster1) in **prod.us-south-3** for **app1.instance1** is feed blocked", sourceCluster1,
                 "memory (usage: 70.0%, feed block limit: 50.0%)", "disk (usage: 95.0%, feed block limit: 90.0%)"));
         assertEquals(expected, curatorDb.readNotifications(tenant));
     }
@@ -230,9 +236,9 @@ public class NotificationsDbTest {
                         "build", reindexingStatus(null, 0.50)))));
         notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(
                 clusterMetrics("cluster1", 0.88, 0.9, 0.3, 0.5), clusterMetrics("cluster2", 0.6, 0.8, 0.9, 0.75), clusterMetrics("cluster3", 0.1, 0.8, 0.2, 0.9)), applicationReindexing1);
-        expected.add(notification(12345, Type.feedBlock, Level.warning, sourceCluster1, "disk (usage: 88.0%, feed block limit: 90.0%)"));
-        expected.add(notification(12345, Type.feedBlock, Level.error, sourceCluster2, "memory (usage: 90.0%, feed block limit: 75.0%)"));
-        expected.add(notification(12345, Type.reindex, Level.info, sourceCluster3, "document type 'announcements' reindexing due to a schema change (75.0% done)", "document type 'build' (50.0% done)"));
+        expected.add(notification(12345, Type.feedBlock, Level.warning, "Cluster [cluster1](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster1) in **prod.us-south-3** for **app1.instance1** is nearly feed blocked", sourceCluster1, "disk (usage: 88.0%, feed block limit: 90.0%)"));
+        expected.add(notification(12345, Type.feedBlock, Level.error, "Cluster [cluster2](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster2) in **prod.us-south-3** for **app1.instance1** is feed blocked", sourceCluster2, "memory (usage: 90.0%, feed block limit: 75.0%)"));
+        expected.add(notification(12345, Type.reindex, Level.info, "Cluster [cluster3](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster3%3Dreindexing) in **prod.us-south-3** for **app1.instance1** is [reindexing](https://docs.vespa.ai/en/operations/reindexing.html)", sourceCluster3, "document type 'announcements' reindexing due to a schema change (75.0% done)", "document type 'build' (50.0% done)"));
         assertEquals(expected, curatorDb.readNotifications(tenant));
 
         // Cluster1 improves, while cluster3 starts having feed block issues and finishes reindexing 'build' documents
@@ -242,12 +248,41 @@ public class NotificationsDbTest {
                         "build", reindexingStatus(null, null)))));
         notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(
                 clusterMetrics("cluster1", 0.15, 0.9, 0.3, 0.5), clusterMetrics("cluster2", 0.6, 0.8, 0.9, 0.75), clusterMetrics("cluster3", 0.78, 0.8, 0.2, 0.9)), applicationReindexing2);
-        expected.set(6, notification(12345, Type.feedBlock, Level.error, sourceCluster2, "memory (usage: 90.0%, feed block limit: 75.0%)"));
-        expected.set(7, notification(12345, Type.feedBlock, Level.warning, sourceCluster3, "disk (usage: 78.0%, feed block limit: 80.0%)"));
-        expected.set(8, notification(12345, Type.reindex, Level.info, sourceCluster3, "document type 'announcements' reindexing due to a schema change (90.0% done)"));
+        expected.set(6, notification(12345, Type.feedBlock, Level.error, "Cluster [cluster2](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster2) in **prod.us-south-3** for **app1.instance1** is feed blocked", sourceCluster2, "memory (usage: 90.0%, feed block limit: 75.0%)"));
+        expected.set(7, notification(12345, Type.feedBlock, Level.warning, "Cluster [cluster3](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster3) in **prod.us-south-3** for **app1.instance1** is nearly feed blocked", sourceCluster3, "disk (usage: 78.0%, feed block limit: 80.0%)"));
+        expected.set(8, notification(12345, Type.reindex, Level.info, "Cluster [cluster3](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster3%3Dreindexing) in **prod.us-south-3** for **app1.instance1** is [reindexing](https://docs.vespa.ai/en/operations/reindexing.html)", sourceCluster3, "document type 'announcements' reindexing due to a schema change (90.0% done)"));
         assertEquals(expected, curatorDb.readNotifications(tenant));
     }
 
+    @Test
+    void title_test() {
+        curatorDb.deleteNotifications(tenant);
+        TenantAndApplicationId tenantApp = TenantAndApplicationId.from(tenant.value(), "app1");
+        ApplicationId app = tenantApp.instance("instance1");
+        ZoneRegistryMock zoneRegistry = new ZoneRegistryMock(SystemName.Public);
+
+        notificationsDb.setApplicationPackageNotification(NotificationSource.from(tenantApp), List.of());
+        notificationsDb.setApplicationPackageNotification(NotificationSource.from(new DeploymentId(app, ZoneId.from("dev.us-east-3"))), List.of());
+        notificationsDb.setSubmissionNotification(tenantApp, "msg");
+        notificationsDb.setTestPackageNotification(tenantApp, List.of());
+        notificationsDb.setDeploymentNotification(new RunId(app, JobType.prod("us-east-3"), 123), "msg");
+        notificationsDb.setDeploymentNotification(new RunId(app, JobType.productionTestOf(ZoneId.from("prod.us-east-3")), 123), "msg");
+        notificationsDb.setDeploymentNotification(new RunId(app, JobType.systemTest(zoneRegistry, CloudName.AWS), 123), "msg");
+        notificationsDb.setDeploymentNotification(new RunId(app, JobType.stagingTest(zoneRegistry, CloudName.AWS), 123), "msg");
+        notificationsDb.setDeploymentNotification(new RunId(app, JobType.dev("us-east-3"), 123), "msg");
+        assertEquals(List.of(
+                "Application package for [app1](https://console.tld/tenant/tenant1/application/app1/prod/instance) has warnings",
+                "Application package for [app1.instance1](https://console.tld/tenant/tenant1/application/app1/dev/instance/instance1) has warnings",
+                "Application package for [app1](https://console.tld/tenant/tenant1/application/app1/prod/instance) has a warning",
+                "There are problems with tests for [app1](https://console.tld/tenant/tenant1/application/app1/prod/instance)",
+                "Deployment job [#123 to us-east-3](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1/job/production-us-east-3/run/123) for application **app1.instance1** has failed",
+                "Test job [#123 to us-east-3](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1/job/test-us-east-3/run/123) for application **app1.instance1** has failed",
+                "[System test #123](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1/job/system-test/run/123) for application **app1.instance1** has failed",
+                "[Staging test #123](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1/job/staging-test/run/123) for application **app1.instance1** has failed",
+                "Deployment job [#123 to dev.us-east-3](https://console.tld/tenant/tenant1/application/app1/dev/instance/instance1/job/dev-us-east-3/run/123) for application **app1.instance1** has failed"
+                ), notificationsDb.listNotifications(NotificationSource.from(tenant), false).stream().map(Notification::title).toList());
+    }
+
     @BeforeEach
     public void init() {
         curatorDb.writeNotifications(tenant, notifications);
@@ -255,12 +290,20 @@ public class NotificationsDbTest {
         mailer.reset();
     }
 
+    private void setNotification(Notification notification) {
+        notificationsDb.setNotification(notification.source(), notification.type(), notification.level(), "", notification.messages(), Optional.empty());
+    }
+
     private static List<Notification> notificationIndices(int... indices) {
         return Arrays.stream(indices).mapToObj(notifications::get).collect(Collectors.toCollection(ArrayList::new));
     }
 
     private static Notification notification(long secondsSinceEpoch, Type type, Level level, NotificationSource source, String... messages) {
-        return new Notification(Instant.ofEpochSecond(secondsSinceEpoch), type, level, source, List.of(messages));
+        return notification(secondsSinceEpoch, type, level, "", source, messages);
+    }
+
+    private static Notification notification(long secondsSinceEpoch, Type type, Level level, String title, NotificationSource source, String... messages) {
+        return new Notification(Instant.ofEpochSecond(secondsSinceEpoch), type, level, source, title, List.of(messages));
     }
 
     private static ClusterMetrics clusterMetrics(String clusterId,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index cc336bfb35b..66fb17410fd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -61,7 +61,6 @@ import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
 import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
-import com.yahoo.vespa.hosted.controller.notification.Notification;
 import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
@@ -1979,15 +1978,11 @@ public class ApplicationApiTest extends ControllerContainerTest {
     }
 
     private void addNotifications(TenantName tenantName) {
-        tester.controller().notificationsDb().setNotification(
+        tester.controller().notificationsDb().setApplicationPackageNotification(
                 NotificationSource.from(TenantAndApplicationId.from(tenantName.value(), "app1")),
-                Notification.Type.applicationPackage,
-                Notification.Level.warning,
-                "Something something deprecated...");
-        tester.controller().notificationsDb().setNotification(
-                NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app2", "instance1"), DeploymentContext.systemTest, 12)),
-                Notification.Type.deployment,
-                Notification.Level.error,
+                List.of("Something something deprecated..."));
+        tester.controller().notificationsDb().setDeploymentNotification(
+                new RunId(ApplicationId.from(tenantName.value(), "app2", "instance1"), DeploymentContext.systemTest, 12),
                 "Failed to deploy: Node allocation failure");
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json
index 44ce2c510f9..6206e3b277a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json
@@ -4,7 +4,7 @@
       "at": 1600000000000,
       "level": "error",
       "type": "deployment",
-      "title": "",
+      "title": "[System test #12](https://console.tld/tenant/tenant1/application/app2/prod/instance/instance1/job/system-test/run/12) for application **app2.instance1** has failed",
       "messages": [
         "Failed to deploy: Node allocation failure"
       ],
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json
index dd8edfcc046..78deea65008 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json
@@ -4,7 +4,7 @@
       "at": 1600000000000,
       "level": "warning",
       "type": "applicationPackage",
-      "title": "",
+      "title": "Application package for [app1](https://console.tld/tenant/tenant1/application/app1/prod/instance) has a warning",
       "messages": [
         "Something something deprecated..."
       ],
@@ -14,7 +14,7 @@
       "at": 1600000000000,
       "level": "error",
       "type": "deployment",
-      "title": "",
+      "title": "[System test #12](https://console.tld/tenant/tenant1/application/app2/prod/instance/instance1/job/system-test/run/12) for application **app2.instance1** has failed",
       "messages": [
         "Failed to deploy: Node allocation failure"
       ],