Merge pull request #18317 from vespa-engine/freva/info-reindexing-notification

Reindexing notification

15 files changed, 154 insertions, 51 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 276eb51981c..818e65b6caf 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -1087,9 +1087,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         }
         ReindexActions reindexActions = actions.getReindexActions();
         if ( ! reindexActions.isEmpty()) {
-            logger.logApplicationPackage(Level.WARNING,
-                                         "Change(s) between active and new application that may require re-index:\n" +
-                                         reindexActions.format());
+            logger.log(Level.WARNING,
+                       "Change(s) between active and new application that may require re-index:\n" +
+                       reindexActions.format());
         }
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java
index 062a21b1f80..cdfdce91500 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java
@@ -40,6 +40,11 @@ public class DeploymentMetricsResponse extends SlimeJsonResponse {
                 metrics.setDouble("diskUtil", disk.util());
                 metrics.setDouble("diskFeedBlockLimit", disk.feedBlockLimit());
             });
+
+            aggregator.reindexingProgress().ifPresent(reindexingProgress -> {
+                Cursor progressObject = cluster.setObject("reindexingProgress");
+                reindexingProgress.forEach(progressObject::setDouble);
+            });
         }
     }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java
index d406cafc3b8..77e2f923d4a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java
@@ -148,6 +148,10 @@ public class ClusterDeploymentMetricsRetriever {
                                         values.field("cluster-controller.resource_usage.memory_limit.last").asDouble())
                                 .addDiskUsage(values.field("cluster-controller.resource_usage.max_disk_utilization.last").asDouble(),
                                         values.field("cluster-controller.resource_usage.disk_limit.last").asDouble()));
+                optionalDouble(values.field("reindexing.progress.last")).ifPresent(progress -> {
+                    if (progress < 0 || progress >= 1) return;
+                    aggregator.get().addReindexingProgress(metric.field("dimensions").field("documenttype").asString(), progress);
+                });
                 break;
         }
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java
index f27cf942dd8..7ce6d84ad8c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java
@@ -1,6 +1,8 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.metrics;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -15,6 +17,7 @@ public class DeploymentMetricsAggregator {
     private Double documentCount;
     private ResourceUsage memoryUsage;
     private ResourceUsage diskUsage;
+    private Map<String, Double> reindexingProgress;
 
     public synchronized DeploymentMetricsAggregator addFeedLatency(double sum, double count) {
         this.feed = combineLatency(this.feed, sum, count);
@@ -46,6 +49,12 @@ public class DeploymentMetricsAggregator {
         return this;
     }
 
+    public synchronized DeploymentMetricsAggregator addReindexingProgress(String documentType, double progress) {
+        if (reindexingProgress == null) this.reindexingProgress = new HashMap<>();
+        this.reindexingProgress.put(documentType, progress);
+        return this;
+    }
+
     public Optional<Double> aggregateFeedLatency() {
         return Optional.ofNullable(feed).map(m -> m.sum / m.count).filter(num -> !num.isNaN());
     }
@@ -80,6 +89,10 @@ public class DeploymentMetricsAggregator {
         return Optional.ofNullable(diskUsage);
     }
 
+    public Optional<Map<String, Double>> reindexingProgress() {
+        return Optional.ofNullable(reindexingProgress);
+    }
+
 
     private static LatencyMetrics combineLatency(LatencyMetrics metricsOrNull, double sum, double count) {
         return Optional.ofNullable(metricsOrNull).orElseGet(LatencyMetrics::new).combine(sum, count);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java
index 7fdfbcdbf03..b5bcae65009 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java
@@ -70,7 +70,9 @@ public class ClusterDeploymentMetricsRetrieverTest {
                 new DeploymentMetricsAggregator()
                         .addDocumentCount(6000.0)
                         .addMemoryUsage(0.89074, 0.8)
-                        .addDiskUsage(0.83517, 0.75),
+                        .addDiskUsage(0.83517, 0.75)
+                        .addReindexingProgress("test_artifacts", 0.71)
+                        .addReindexingProgress("announcements", 0),
                 aggregatorMap.get(expectedContentCluster)
         );
 
@@ -113,6 +115,7 @@ public class ClusterDeploymentMetricsRetrieverTest {
         compareOptionals(expected.diskUsage(), actual.diskUsage(), (a, b) -> assertDoubles.accept(a.feedBlockLimit(), b.feedBlockLimit()));
         compareOptionals(expected.memoryUsage(), actual.memoryUsage(), (a, b) -> assertDoubles.accept(a.util(), b.util()));
         compareOptionals(expected.memoryUsage(), actual.memoryUsage(), (a, b) -> assertDoubles.accept(a.feedBlockLimit(), b.feedBlockLimit()));
+        assertEquals(expected.reindexingProgress(), actual.reindexingProgress());
     }
 
     @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
diff --git a/configserver/src/test/resources/metrics/clustercontroller_metrics.json b/configserver/src/test/resources/metrics/clustercontroller_metrics.json
index 9afcb34d77d..65468749940 100644
--- a/configserver/src/test/resources/metrics/clustercontroller_metrics.json
+++ b/configserver/src/test/resources/metrics/clustercontroller_metrics.json
@@ -20,6 +20,48 @@
         },
         {
           "values": {
+            "reindexing.progress.last": 0.71
+          },
+          "dimensions": {
+            "clustertype": "content",
+            "clusterid": "content_cluster_id",
+            "documenttype": "test_artifacts"
+          }
+        },
+        {
+          "values": {
+            "reindexing.progress.last": 1
+          },
+          "dimensions": {
+            "clustertype": "content",
+            "clusterid": "content_cluster_id",
+            "documenttype": "builds"
+          }
+        },
+        {
+          "values": {
+            "reindexing.progress.last": 0
+          },
+          "dimensions": {
+            "clustertype": "content",
+            "clusterid": "content_cluster_id",
+            "documenttype": "announcements",
+            "state": "running"
+          }
+        },
+        {
+          "values": {
+            "reindexing.progress.last": -1
+          },
+          "dimensions": {
+            "clustertype": "content",
+            "clusterid": "content_cluster_id",
+            "documenttype": "announcements",
+            "state": "successful"
+          }
+        },
+        {
+          "values": {
             "some.other.metrics": 1
           },
           "dimensions": {
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java
index 0e11bcdccaf..ee74aca0e14 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java
@@ -23,11 +23,13 @@ public class ClusterMetrics {
     private final String clusterId;
     private final String clusterType;
     private final Map<String, Double> metrics;
+    private final Map<String, Double> reindexingProgress;
 
-    public ClusterMetrics(String clusterId, String clusterType, Map<String, Double> metrics) {
+    public ClusterMetrics(String clusterId, String clusterType, Map<String, Double> metrics, Map<String, Double> reindexingProgress) {
         this.clusterId = clusterId;
         this.clusterType = clusterType;
         this.metrics = Map.copyOf(metrics);
+        this.reindexingProgress = Map.copyOf(reindexingProgress);
     }
 
     public String getClusterId() {
@@ -74,4 +76,7 @@ public class ClusterMetrics {
         return Optional.ofNullable(metrics.get(DISK_FEED_BLOCK_LIMIT));
     }
 
+    public Map<String, Double> reindexingProgress() {
+        return reindexingProgress;
+    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index 20154c4f122..ba4aaf92fc8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -69,7 +69,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer {
                                                                lockedInstance -> lockedInstance.with(existingDeployment.zone(), newMetrics)
                                                                                                .recordActivityAt(now, existingDeployment.zone())));
 
-                                controller().notificationsDb().setDeploymentFeedingBlockedNotifications(deploymentId, clusterMetrics);
+                                controller().notificationsDb().setDeploymentMetricsNotifications(deploymentId, clusterMetrics);
                             });
                         } catch (Exception e) {
                             failures.incrementAndGet();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
index ea0422ea9fc..b65a9290e43 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
@@ -62,7 +62,7 @@ public class Notification {
 
     public enum Level {
         // Must be ordered in order of importance
-        warning, error
+        info, warning, error
     }
 
     public enum Type {
@@ -73,7 +73,10 @@ public class Notification {
         deployment,
 
         /** Application cluster is (near) external feed blocked */
-        feedBlock;
+        feedBlock,
+
+        /** Application cluster is reindexing document(s) */
+        reindex;
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java
index beb771b1aa2..7c2d990750c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java
@@ -124,31 +124,22 @@ public class NotificationsDb {
     }
 
     /**
-     * Updates feeding blocked notifications for the given deployment based on current cluster metrics.
-     * Will clear notifications of any cluster not reporting the metrics or whose metrics indicate feed is not blocked,
-     * while setting notifications for cluster that are (Level.error) or are nearly (Level.warning) feed blocked.
+     * Updates notifications based on deployment metrics (e.g. feed blocked and reindexing progress) for the given
+     * deployment based on current cluster metrics.
+     * Will clear notifications of any cluster not reporting the metrics or whose metrics indicate feed is not blocked
+     * or reindexing no longer in progress. Will set notification for clusters:
+     *  - that are (Level.error) or are nearly (Level.warning) feed blocked,
+     *  - that are (Level.info) currently reindexing at least 1 document type.
      */
-    public void setDeploymentFeedingBlockedNotifications(DeploymentId deploymentId, List<ClusterMetrics> clusterMetrics) {
+    public void setDeploymentMetricsNotifications(DeploymentId deploymentId, List<ClusterMetrics> clusterMetrics) {
         Instant now = clock.instant();
-        List<Notification> feedBlockNotifications = clusterMetrics.stream()
+        List<Notification> newNotifications = clusterMetrics.stream()
                 .flatMap(metric -> {
-                    Optional<Pair<Level, String>> memoryStatus =
-                            resourceUtilToFeedBlockStatus("memory", metric.memoryUtil(), metric.memoryFeedBlockLimit());
-                    Optional<Pair<Level, String>> diskStatus =
-                            resourceUtilToFeedBlockStatus("disk", metric.diskUtil(), metric.diskFeedBlockLimit());
-                    if (memoryStatus.isEmpty() && diskStatus.isEmpty()) return Stream.empty();
-
-                    // Find the max among levels
-                    Level level = Stream.of(memoryStatus, diskStatus)
-                            .flatMap(status -> status.stream().map(Pair::getFirst))
-                            .max(Comparator.comparing(Enum::ordinal)).get();
-                    List<String> messages = Stream.concat(memoryStatus.stream(), diskStatus.stream())
-                            .filter(status -> status.getFirst() == level) // Do not mix message from different levels
-                            .map(Pair::getSecond)
-                            .collect(Collectors.toUnmodifiableList());
                     NotificationSource source = NotificationSource.from(deploymentId, ClusterSpec.Id.from(metric.getClusterId()));
-                    return Stream.of(new Notification(now, Type.feedBlock, level, source, messages));
+                    return Stream.of(createFeedBlockNotification(source, now, metric),
+                                     createReindexNotification(source, now, metric));
                 })
+                .flatMap(Optional::stream)
                 .collect(Collectors.toUnmodifiableList());
 
         NotificationSource deploymentSource = NotificationSource.from(deploymentId);
@@ -157,10 +148,11 @@ public class NotificationsDb {
             List<Notification> updated = Stream.concat(
                     initial.stream()
                             .filter(notification ->
-                                    // Filter out old feed block notifications for this deployment
-                                    notification.type() != Type.feedBlock || !deploymentSource.contains(notification.source())),
+                                    // Filter out old feed block notifications and reindex for this deployment
+                                    (notification.type() != Type.feedBlock && notification.type() != Type.reindex) ||
+                                            !deploymentSource.contains(notification.source())),
                     // ... and add the new notifications for this deployment
-                    feedBlockNotifications.stream())
+                    newNotifications.stream())
                     .collect(Collectors.toUnmodifiableList());
 
             if (!initial.equals(updated))
@@ -168,6 +160,33 @@ public class NotificationsDb {
         }
     }
 
+    private static Optional<Notification> createFeedBlockNotification(NotificationSource source, Instant at, ClusterMetrics metric) {
+        Optional<Pair<Level, String>> memoryStatus =
+                resourceUtilToFeedBlockStatus("memory", metric.memoryUtil(), metric.memoryFeedBlockLimit());
+        Optional<Pair<Level, String>> diskStatus =
+                resourceUtilToFeedBlockStatus("disk", metric.diskUtil(), metric.diskFeedBlockLimit());
+        if (memoryStatus.isEmpty() && diskStatus.isEmpty()) return Optional.empty();
+
+        // Find the max among levels
+        Level level = Stream.of(memoryStatus, diskStatus)
+                .flatMap(status -> status.stream().map(Pair::getFirst))
+                .max(Comparator.comparing(Enum::ordinal)).get();
+        List<String> messages = Stream.concat(memoryStatus.stream(), diskStatus.stream())
+                .filter(status -> status.getFirst() == level) // Do not mix message from different levels
+                .map(Pair::getSecond)
+                .collect(Collectors.toUnmodifiableList());
+        return Optional.of(new Notification(at, Type.feedBlock, level, source, messages));
+    }
+
+    private static Optional<Notification> createReindexNotification(NotificationSource source, Instant at, ClusterMetrics metric) {
+        if (metric.reindexingProgress().isEmpty()) return Optional.empty();
+        List<String> messages = metric.reindexingProgress().entrySet().stream()
+                .map(entry -> String.format("document type '%s' (%.1f%% done)", entry.getKey(), 100 * entry.getValue()))
+                .sorted()
+                .collect(Collectors.toUnmodifiableList());
+        return Optional.of(new Notification(at, Type.reindex, Level.info, source, messages));
+    }
+
     /**
      * Returns a feed block summary for the given resource: the notification level and
      * notification message for the given resource utilization wrt. given resource limit.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java
index 54dc102d573..06263329091 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java
@@ -93,6 +93,7 @@ public class NotificationsSerializer {
             case applicationPackage: return "applicationPackage";
             case deployment: return "deployment";
             case feedBlock: return "feedBlock";
+            case reindex: return "reindex";
             default: throw new IllegalArgumentException("No serialization defined for notification type " + type);
         }
     }
@@ -102,12 +103,14 @@ public class NotificationsSerializer {
             case "applicationPackage": return Notification.Type.applicationPackage;
             case "deployment": return Notification.Type.deployment;
             case "feedBlock": return Notification.Type.feedBlock;
+            case "reindex": return Notification.Type.reindex;
             default: throw new IllegalArgumentException("Unknown serialized notification type value '" + field.asString() + "'");
         }
     }
 
     private static String asString(Notification.Level level) {
         switch (level) {
+            case info: return "info";
             case warning: return "warning";
             case error: return "error";
             default: throw new IllegalArgumentException("No serialization defined for notification level " + level);
@@ -116,6 +119,7 @@ public class NotificationsSerializer {
 
     private static Notification.Level levelFrom(Inspector field) {
         switch (field.asString()) {
+            case "info": return Notification.Level.info;
             case "warning": return Notification.Level.warning;
             case "error": return Notification.Level.error;
             default: throw new IllegalArgumentException("Unknown serialized notification level value '" + field.asString() + "'");
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 017da94facc..937d3d77fae 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -525,12 +525,14 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
             case applicationPackage: return "applicationPackage";
             case deployment: return "deployment";
             case feedBlock: return "feedBlock";
+            case reindex: return "reindex";
             default: throw new IllegalArgumentException("No serialization defined for notification type " + type);
         }
     }
 
     private static String notificationLevelAsString(Notification.Level level) {
         switch (level) {
+            case info: return "info";
             case warning: return "warning";
             case error: return "error";
             default: throw new IllegalArgumentException("No serialization defined for notification level " + level);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index 59fb5b596f1..c45aaa563e1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -118,8 +118,8 @@ public class DeploymentMetricsMaintainerTest {
     @Test
     public void cluster_metric_aggregation_test() {
         List<ClusterMetrics> clusterMetrics = List.of(
-                new ClusterMetrics("niceCluster", "container", Map.of("queriesPerSecond", 23.0, "queryLatency", 1337.0)),
-                new ClusterMetrics("alsoNiceCluster", "container", Map.of("queriesPerSecond", 11.0, "queryLatency", 12.0)));
+                new ClusterMetrics("niceCluster", "container", Map.of("queriesPerSecond", 23.0, "queryLatency", 1337.0), Map.of()),
+                new ClusterMetrics("alsoNiceCluster", "container", Map.of("queriesPerSecond", 11.0, "queryLatency", 12.0), Map.of()));
 
         DeploymentMetrics deploymentMetrics = DeploymentMetricsMaintainer.updateDeploymentMetrics(DeploymentMetrics.none, clusterMetrics);
 
@@ -131,7 +131,7 @@ public class DeploymentMetricsMaintainerTest {
     }
 
     private void setMetrics(ApplicationId application, Map<String, Double> metrics) {
-        var clusterMetrics = new ClusterMetrics("default", "container", metrics);
+        var clusterMetrics = new ClusterMetrics("default", "container", metrics, Map.of());
         tester.controllerTester().serviceRegistry().configServerMock().setMetrics(new DeploymentId(application, ZoneId.from("dev", "us-east-1")), clusterMetrics);
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java
index 7b4882de3ff..29d77c38b1a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java
@@ -16,7 +16,6 @@ import java.time.Duration;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 /**
  * Tests the traffic fraction updater. This also tests its dependency on DeploymentMetricsMaintainer.
@@ -82,7 +81,7 @@ public class TrafficShareUpdaterTest {
     }
 
     private void setQpsMetric(double qps, ApplicationId application, ZoneId zone, DeploymentTester tester) {
-        var clusterMetrics = new ClusterMetrics("default", "container", Map.of(ClusterMetrics.QUERIES_PER_SECOND, qps));
+        var clusterMetrics = new ClusterMetrics("default", "container", Map.of(ClusterMetrics.QUERIES_PER_SECOND, qps), Map.of());
         tester.controllerTester().serviceRegistry().configServerMock().setMetrics(new DeploymentId(application, zone), clusterMetrics);
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
index f6c17ffcef1..454a4f81524 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
@@ -107,54 +107,56 @@ public class NotificationsDbTest {
         List<Notification> expected = new ArrayList<>(notifications);
 
         // No metrics, no new notification
-        notificationsDb.setDeploymentFeedingBlockedNotifications(deploymentId, List.of());
+        notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of());
         assertEquals(expected, curatorDb.readNotifications(tenant));
 
         // Metrics that contain none of the feed block metrics does not create new notification
-        notificationsDb.setDeploymentFeedingBlockedNotifications(deploymentId, List.of(clusterMetrics("cluster1", null, null, null, null)));
+        notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", null, null, null, null, Map.of())));
         assertEquals(expected, curatorDb.readNotifications(tenant));
 
         // Metrics that only contain util or limit (should not be possible) should not cause any issues
-        notificationsDb.setDeploymentFeedingBlockedNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, null, null, 0.5)));
+        notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, null, null, 0.5, Map.of())));
         assertEquals(expected, curatorDb.readNotifications(tenant));
 
         // One resource is at warning
-        notificationsDb.setDeploymentFeedingBlockedNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.85, 0.9, 0.3, 0.5)));
+        notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.85, 0.9, 0.3, 0.5, Map.of())));
         expected.add(notification(12345, Type.feedBlock, Level.warning, sourceCluster1, "disk (usage: 85.0%, feed block limit: 90.0%)"));
         assertEquals(expected, curatorDb.readNotifications(tenant));
 
         // Both resources over the limit
-        notificationsDb.setDeploymentFeedingBlockedNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, 0.9, 0.3, 0.5)));
+        notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, 0.9, 0.3, 0.5, Map.of())));
         expected.set(6, notification(12345, Type.feedBlock, Level.error, sourceCluster1, "disk (usage: 95.0%, feed block limit: 90.0%)"));
         assertEquals(expected, curatorDb.readNotifications(tenant));
 
         // One resource at warning, one at error: Only show error message
-        notificationsDb.setDeploymentFeedingBlockedNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, 0.9, 0.7, 0.5)));
+        notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, 0.9, 0.7, 0.5, Map.of())));
         expected.set(6, notification(12345, Type.feedBlock, Level.error, sourceCluster1,
                 "memory (usage: 70.0%, feed block limit: 50.0%)", "disk (usage: 95.0%, feed block limit: 90.0%)"));
         assertEquals(expected, curatorDb.readNotifications(tenant));
     }
 
     @Test
-    public void feed_blocked_multiple_cluster_test() {
+    public void deployment_metrics_multiple_cluster_test() {
         DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenant.value(), "app1", "instance1"), ZoneId.from("prod", "us-south-3"));
         NotificationSource sourceCluster1 = NotificationSource.from(deploymentId, ClusterSpec.Id.from("cluster1"));
         NotificationSource sourceCluster2 = NotificationSource.from(deploymentId, ClusterSpec.Id.from("cluster2"));
         NotificationSource sourceCluster3 = NotificationSource.from(deploymentId, ClusterSpec.Id.from("cluster3"));
         List<Notification> expected = new ArrayList<>(notifications);
 
-        // Cluster1 and cluster2 are having issues
-        notificationsDb.setDeploymentFeedingBlockedNotifications(deploymentId, List.of(
-                clusterMetrics("cluster1", 0.85, 0.9, 0.3, 0.5), clusterMetrics("cluster2", 0.6, 0.8, 0.9, 0.75), clusterMetrics("cluster3", 0.1, 0.8, 0.2, 0.9)));
+        // Cluster1 and cluster2 are having feed block issues, cluster 3 is reindexing
+        notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(
+                clusterMetrics("cluster1", 0.85, 0.9, 0.3, 0.5, Map.of()), clusterMetrics("cluster2", 0.6, 0.8, 0.9, 0.75, Map.of()), clusterMetrics("cluster3", 0.1, 0.8, 0.2, 0.9, Map.of("announcements", 0.75, "build", 0.5))));
         expected.add(notification(12345, Type.feedBlock, Level.warning, sourceCluster1, "disk (usage: 85.0%, feed block limit: 90.0%)"));
         expected.add(notification(12345, Type.feedBlock, Level.error, sourceCluster2, "memory (usage: 90.0%, feed block limit: 75.0%)"));
+        expected.add(notification(12345, Type.reindex, Level.info, sourceCluster3, "document type 'announcements' (75.0% done)", "document type 'build' (50.0% done)"));
         assertEquals(expected, curatorDb.readNotifications(tenant));
 
-        // Cluster1 improves, while cluster3 starts having issues
-        notificationsDb.setDeploymentFeedingBlockedNotifications(deploymentId, List.of(
-                clusterMetrics("cluster1", 0.15, 0.9, 0.3, 0.5), clusterMetrics("cluster2", 0.6, 0.8, 0.9, 0.75), clusterMetrics("cluster3", 0.75, 0.8, 0.2, 0.9)));
+        // Cluster1 improves, while cluster3 starts having feed block issues and finishes reindexing 'build' documents
+        notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(
+                clusterMetrics("cluster1", 0.15, 0.9, 0.3, 0.5, Map.of()), clusterMetrics("cluster2", 0.6, 0.8, 0.9, 0.75, Map.of()), clusterMetrics("cluster3", 0.75, 0.8, 0.2, 0.9,  Map.of("announcements", 0.9))));
         expected.set(6, notification(12345, Type.feedBlock, Level.error, sourceCluster2, "memory (usage: 90.0%, feed block limit: 75.0%)"));
         expected.set(7, notification(12345, Type.feedBlock, Level.warning, sourceCluster3, "disk (usage: 75.0%, feed block limit: 80.0%)"));
+        expected.set(8, notification(12345, Type.reindex, Level.info, sourceCluster3, "document type 'announcements' (90.0% done)"));
         assertEquals(expected, curatorDb.readNotifications(tenant));
     }
 
@@ -206,12 +208,14 @@ public class NotificationsDbTest {
         return new Notification(Instant.ofEpochSecond(secondsSinceEpoch), type, level, source, List.of(messages));
     }
 
-    private static ClusterMetrics clusterMetrics(String clusterId, Double diskUtil, Double diskLimit, Double memoryUtil, Double memoryLimit) {
+    private static ClusterMetrics clusterMetrics(String clusterId,
+                                                 Double diskUtil, Double diskLimit, Double memoryUtil, Double memoryLimit,
+                                                 Map<String, Double> reindexingProgress) {
         Map<String, Double> metrics = new HashMap<>();
         if (diskUtil != null) metrics.put(ClusterMetrics.DISK_UTIL, diskUtil);
         if (diskLimit != null) metrics.put(ClusterMetrics.DISK_FEED_BLOCK_LIMIT, diskLimit);
         if (memoryUtil != null) metrics.put(ClusterMetrics.MEMORY_UTIL, memoryUtil);
         if (memoryLimit != null) metrics.put(ClusterMetrics.MEMORY_FEED_BLOCK_LIMIT, memoryLimit);
-        return new ClusterMetrics(clusterId, "content", metrics);
+        return new ClusterMetrics(clusterId, "content", metrics, reindexingProgress);
     }
 }