Merge pull request #21976 from vespa-engine/ean/notify-on-notification

Use email as notification sink for newly created notifications

4 files changed, 150 insertions, 6 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 6afeab9b4e6..c35e8c5a7ac 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.config.ControllerConfig;
 import com.yahoo.vespa.hosted.controller.deployment.JobController;
 import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder;
 import com.yahoo.vespa.hosted.controller.notification.NotificationsDb;
+import com.yahoo.vespa.hosted.controller.notify.Notifier;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.persistence.JobControlFlags;
 import com.yahoo.vespa.hosted.controller.security.AccessControl;
@@ -88,6 +89,7 @@ public class Controller extends AbstractComponent {
     private final CuratorArchiveBucketDb archiveBucketDb;
     private final NotificationsDb notificationsDb;
     private final SupportAccessControl supportAccessControl;
+    private final Notifier notifier;
 
     /**
      * Creates a controller 
@@ -126,6 +128,7 @@ public class Controller extends AbstractComponent {
         auditLogger = new AuditLogger(curator, clock);
         jobControl = new JobControl(new JobControlFlags(curator, flagSource));
         archiveBucketDb = new CuratorArchiveBucketDb(this);
+        notifier = new Notifier(curator, serviceRegistry.mailer());
         notificationsDb = new NotificationsDb(this);
         supportAccessControl = new SupportAccessControl(this);
 
@@ -330,4 +333,7 @@ public class Controller extends AbstractComponent {
         return supportAccessControl;
     }
 
+    public Notifier notifier() {
+        return notifier;
+    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java
index c0bd1ac03ff..5244d46d0a9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.notify.Notifier;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
 import java.time.Clock;
@@ -32,14 +33,16 @@ public class NotificationsDb {
 
     private final Clock clock;
     private final CuratorDb curatorDb;
+    private final Notifier notifier;
 
     public NotificationsDb(Controller controller) {
-        this(controller.clock(), controller.curator());
+        this(controller.clock(), controller.curator(), controller.notifier());
     }
 
-    NotificationsDb(Clock clock, CuratorDb curatorDb) {
+    NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier) {
         this.clock = clock;
         this.curatorDb = curatorDb;
+        this.notifier = notifier;
     }
 
     public List<TenantName> listTenantsWithNotifications() {
@@ -61,13 +64,24 @@ public class NotificationsDb {
      * already exists, it'll be replaced by this one instead
      */
     public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) {
+        Optional<Notification> changed = Optional.empty();
         try (Lock lock = curatorDb.lockNotifications(source.tenant())) {
-            List<Notification> notifications = curatorDb.readNotifications(source.tenant()).stream()
+            var existingNotifications = curatorDb.readNotifications(source.tenant());
+            List<Notification> notifications = existingNotifications.stream()
                     .filter(notification -> !source.equals(notification.source()) || type != notification.type())
                     .collect(Collectors.toCollection(ArrayList::new));
-            notifications.add(new Notification(clock.instant(), type, level, source, messages));
+            var notification = new Notification(clock.instant(), type, level, source, messages);
+            // Be conservative for now, only dispatch notifications if they are from new source or with new type.
+            // the message content and level is ignored for now
+            if (!existingNotifications.stream().anyMatch(n -> n.source().equals(source) && n.type().equals(type))) {
+                changed = Optional.of(notification);
+            }
+            notifications.add(notification);
             curatorDb.writeNotifications(source.tenant(), notifications);
         }
+        if (changed.isPresent()) {
+           notifier.dispatch(changed.get());
+        }
     }
 
     /** Remove the notification with the given source and type */
@@ -131,8 +145,9 @@ public class NotificationsDb {
                     newNotifications.stream())
                     .collect(Collectors.toUnmodifiableList());
 
-            if (!initial.equals(updated))
+            if (!initial.equals(updated)) {
                 curatorDb.writeNotifications(deploymentSource.tenant(), updated);
+            }
         }
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java
new file mode 100644
index 00000000000..46e1fd904ed
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java
@@ -0,0 +1,76 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.notify;
+
+import com.yahoo.text.Text;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException;
+import com.yahoo.vespa.hosted.controller.notification.Notification;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
+import com.yahoo.vespa.hosted.controller.tenant.TenantContacts;
+
+import java.util.Collection;
+import java.util.Objects;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * Notifier is responsible for dispatching user notifications to their chosen Contact points.
+ *
+ * @author enygaard
+ */
+public class Notifier {
+    private final CuratorDb curatorDb;
+    private final Mailer mailer;
+
+    private static final Logger log = Logger.getLogger(Notifier.class.getName());
+
+    public Notifier(CuratorDb curatorDb, Mailer mailer) {
+        this.curatorDb = Objects.requireNonNull(curatorDb);
+        this.mailer = Objects.requireNonNull(mailer);
+    }
+
+    public void dispatch(Notification notification) {
+        var tenant = curatorDb.readTenant(notification.source().tenant());
+        tenant.stream().forEach(t -> {
+            if (t instanceof CloudTenant) {
+                var ct = (CloudTenant) t;
+                ct.info().contacts().all().stream()
+                        .filter(c -> c.audiences().contains(TenantContacts.Audience.NOTIFICATIONS))
+                        .collect(Collectors.groupingBy(TenantContacts.Contact::type, Collectors.toList()))
+                        .entrySet()
+                        .forEach(e -> dispatch(notification, e.getKey(), e.getValue()));
+            }
+        });
+    }
+
+    private void dispatch(Notification notification, TenantContacts.Type type, Collection<? extends TenantContacts.Contact> contacts) {
+        switch (type) {
+            case EMAIL:
+                dispatch(notification, contacts.stream().map(c -> (TenantContacts.EmailContact) c).collect(Collectors.toList()));
+                break;
+            default:
+                throw new IllegalArgumentException("Unknown TenantContacts type " + type.name());
+        }
+    }
+
+    private void dispatch(Notification notification, Collection<TenantContacts.EmailContact> contacts) {
+        try {
+            mailer.send(mailOf(notification, contacts.stream().map(c -> c.email()).collect(Collectors.toList())));
+        } catch (MailerException e) {
+            log.log(Level.SEVERE, "Failed sending email", e);
+        }
+    }
+
+    private Mail mailOf(Notification n, Collection<String> recipients) {
+        var subject = Text.format("[%s] Vespa Notification for %s", n.level().toString().toUpperCase(), n.type().name());
+        var body = new StringBuilder();
+        body.append("Source: ").append(n.source().toString()).append("\n")
+                .append("\n")
+                .append(String.join("\n", n.messages()));
+        return new Mail(recipients, subject.toString(), body.toString());
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
index d51856b329d..a5655d2e6eb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.notification;
 
+import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.TenantName;
@@ -11,8 +12,14 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.notify.Notifier;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
+import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo;
+import com.yahoo.vespa.hosted.controller.tenant.TenantContacts;
+import com.yahoo.vespa.hosted.controller.tenant.TenantInfo;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -22,6 +29,7 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.hosted.controller.notification.Notification.Level;
@@ -36,6 +44,19 @@ import static org.junit.Assert.assertTrue;
 public class NotificationsDbTest {
 
     private static final TenantName tenant = TenantName.from("tenant1");
+    private static final String email = "user1@example.com";
+    private static final CloudTenant cloudTenant = new CloudTenant(tenant,
+            Instant.now(),
+            LastLoginInfo.EMPTY,
+            Optional.empty(),
+            ImmutableBiMap.of(),
+            TenantInfo.empty()
+                    .withContacts(new TenantContacts(
+                            List.of(new TenantContacts.EmailContact(
+                                List.of(TenantContacts.Audience.NOTIFICATIONS),
+                            email)))),
+            List.of(),
+            Optional.empty());
     private static final List<Notification> notifications = List.of(
             notification(1001, Type.deployment, Level.error, NotificationSource.from(tenant), "tenant msg"),
             notification(1101, Type.applicationPackage, Level.warning, NotificationSource.from(TenantAndApplicationId.from(tenant.value(), "app1")), "app msg"),
@@ -46,7 +67,8 @@ public class NotificationsDbTest {
 
     private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(12345));
     private final MockCuratorDb curatorDb = new MockCuratorDb();
-    private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb);
+    private final MockMailer mailer = new MockMailer();
+    private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, mailer));
 
     @Test
     public void list_test() {
@@ -75,6 +97,29 @@ public class NotificationsDbTest {
     }
 
     @Test
+    public void notifier_test() {
+        Notification notification1 = notification(12345, Type.deployment, Level.warning, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg #2");
+        Notification notification2 = notification(12345, Type.deployment, Level.error,   NotificationSource.from(ApplicationId.from(tenant.value(), "app3", "instance2")), "instance msg #3");
+        Notification notification3 = notification(12345, Type.reindex, Level.warning, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg #2");
+
+        var a = notifications.get(0);
+        notificationsDb.setNotification(a.source(), a.type(), a.level(), a.messages());
+        assertEquals(0, mailer.inbox(email).size());
+
+        // Replace the 3rd notification. but don't change source or type
+        notificationsDb.setNotification(notification1.source(), notification1.type(), notification1.level(), notification1.messages());
+        assertEquals(0, mailer.inbox(email).size());
+
+        // Notification for a new app, add without replacement
+        notificationsDb.setNotification(notification2.source(), notification2.type(), notification2.level(), notification2.messages());
+        assertEquals(1, mailer.inbox(email).size());
+
+        // Notification for new type on existing app
+        notificationsDb.setNotification(notification3.source(), notification3.type(), notification3.level(), notification3.messages());
+        assertEquals(2, mailer.inbox(email).size());
+    }
+
+    @Test
     public void remove_single_test() {
         // Remove the 3rd notification
         notificationsDb.removeNotification(NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), Type.deployment);
@@ -160,6 +205,8 @@ public class NotificationsDbTest {
     @Before
     public void init() {
         curatorDb.writeNotifications(tenant, notifications);
+        curatorDb.writeTenant(cloudTenant);
+        mailer.reset();
     }
 
     private static List<Notification> notificationIndices(int... indices) {