Add notifications on application package warnings and deployment failures

2 files changed, 51 insertions, 2 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 7df696e424a..0d6ec682a19 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -11,6 +11,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
@@ -57,6 +58,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
 import com.yahoo.vespa.hosted.controller.deployment.JobStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
+import com.yahoo.vespa.hosted.controller.notification.Notification;
 import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.security.AccessControl;
@@ -85,6 +87,7 @@ import java.util.TreeMap;
 import java.util.function.Consumer;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved;
@@ -391,6 +394,16 @@ public class ApplicationController {
             // Record the quota usage for this application
             var quotaUsage = deploymentQuotaUsage(zone, job.application());
 
+            // For direct deployments use the full application ID, but otherwise use just the tenant and application as
+            // the source since it's the same application, so it should have the same warnings
+            NotificationSource source = zone.environment().isManuallyDeployed() ?
+                    NotificationSource.from(job.application()) : NotificationSource.from(applicationId);
+            List<String> warnings = Optional.ofNullable(result.prepareResponse().log)
+                    .map(logs -> logs.stream().filter(log -> LogLevel.parse(log.level).intValue() >= Level.WARNING.intValue()).map(log -> log.message).collect(Collectors.toList()))
+                    .orElseGet(List::of);
+            if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.APPLICATION_PACKAGE_WARNING);
+            else controller.notificationsDb().addNotification(source, Notification.Type.APPLICATION_PACKAGE_WARNING, warnings);
+
             lockApplicationOrThrow(applicationId, application ->
                     store(application.with(job.application().instance(),
                                            instance -> instance.withNewDeployment(zone, revision, platform,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index d7465facd72..a0bbcc0249b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -46,6 +46,8 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateException;
 import com.yahoo.vespa.hosted.controller.config.ControllerConfig;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
+import com.yahoo.vespa.hosted.controller.notification.Notification;
+import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
 import com.yahoo.yolean.Exceptions;
 
@@ -67,6 +69,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -671,7 +674,8 @@ public class InternalStepRunner implements StepRunner {
         try {
             controller.jobController().active(id).ifPresent(run -> {
                 if (run.hasFailed())
-                    sendNotification(run, logger);
+                    sendEmailNotification(run, logger);
+                updateConsoleNotification(run);
             });
         }
         catch (IllegalStateException e) {
@@ -682,7 +686,7 @@ public class InternalStepRunner implements StepRunner {
     }
 
     /** Sends a mail with a notification of a failed run, if one should be sent. */
-    private void sendNotification(Run run, DualLogger logger) {
+    private void sendEmailNotification(Run run, DualLogger logger) {
         Application application = controller.applications().requireApplication(TenantAndApplicationId.from(run.id().application()));
         Notifications notifications = application.deploymentSpec().requireInstance(run.id().application().instance()).notifications();
         boolean newCommit = application.require(run.id().application().instance()).change().application()
@@ -706,6 +710,38 @@ public class InternalStepRunner implements StepRunner {
         }
     }
 
+    private void updateConsoleNotification(Run run) {
+        NotificationSource source = NotificationSource.from(run.id());
+        Consumer<String> updater = msg -> controller.notificationsDb().addNotification(source, Notification.Type.DEPLOYMENT_FAILURE, msg);
+        switch (run.status()) {
+            case running:
+            case aborted:
+                return; // If running, its too early to update. If aborted, let's wait and see how the next run goes.
+            case success:
+                controller.notificationsDb().removeNotification(source, Notification.Type.DEPLOYMENT_FAILURE);
+                return;
+            case outOfCapacity:
+                if (run.id().type().isProduction()) updater.accept("due to lack of capacity. Please contact the Vespa team to request more!");
+                return;
+            case deploymentFailed:
+                updater.accept("due to an invalid application configuration, or timeout of other deployments of the same application");
+                return;
+            case installationFailed:
+                updater.accept("as nodes were not able to start the new Java containers");
+                return;
+            case testFailure:
+                updater.accept("one or more verification tests against the deployment failed");
+                return;
+            case error:
+            case endpointCertificateTimeout:
+                break;
+            default:
+                logger.log(WARNING, "Don't know what to set console notification to for run status '" + run.status() + "'");
+        }
+        updater.accept("something in the framework went wrong. Such errors are " +
+                "usually transient. Please contact the Vespa team if the problem persists!");
+    }
+
     private Optional<Mail> mailOf(Run run, List<String> recipients) {
         switch (run.status()) {
             case running: