Merge pull request #24299 from vespa-engine/mpolden/rebuild-improvements

Rebuild improvements

1 files changed, 7 insertions, 0 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
index 5f43d80b87a..822ed338b56 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
@@ -158,10 +158,17 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
         for (var host : softRebuildingHosts) {
             Optional<NodeMutex> optionalMutex = nodeRepository().nodes().lockAndGet(host, Optional.of(Duration.ofSeconds(10)));
             try (NodeMutex mutex = optionalMutex.get()) {
+                // Re-check flag while holding lock
+                host = mutex.node();
+                if (!host.status().wantToRebuild()) {
+                    continue;
+                }
                 Node updatedNode = hostProvisioner.replaceRootDisk(host);
                 if (!updatedNode.status().wantToRebuild()) {
                     nodeRepository().nodes().write(updatedNode, mutex);
                 }
+            } catch (RuntimeException e) {
+                log.log(Level.WARNING, "Failed to rebuild " + host.hostname() + ": " + Exceptions.toMessageString(e) + ", will retry in " + interval());
             }
         }
     }