Download packages in parallel in maintainer

1 files changed, 46 insertions, 30 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
index b2c489cf074..b85f9376f61 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
@@ -20,8 +20,11 @@ import com.yahoo.vespa.filedistribution.FileReferenceDownload;
 
 import java.io.File;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.Future;
+import java.util.function.Supplier;
 import java.util.logging.Logger;
 
 import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk;
@@ -52,44 +55,57 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
     @Override
     protected double maintain() {
         int attempts = 0;
-        int failures = 0;
+        int[] failures = new int[1];
 
+        List<Runnable> futureDownloads = new ArrayList<>();
         for (TenantName tenantName : applicationRepository.tenantRepository().getAllTenantNames())
             for (Session session : applicationRepository.tenantRepository().getTenant(tenantName).getSessionRepository().getRemoteSessions()) {
-            if (shuttingDown())
-                break;
+                if (shuttingDown())
+                    break;
 
-            switch (session.getStatus()) {
-                case PREPARE, ACTIVATE: break;
-                default: continue;
-            }
+                switch (session.getStatus()) {
+                    case PREPARE, ACTIVATE: break;
+                    default: continue;
+                }
 
-            var applicationId = session.getApplicationId();
-            log.finest(() -> "Verifying application package for " + applicationId);
-
-            Optional<FileReference> appFileReference = session.getApplicationPackageReference();
-            if (appFileReference.isPresent()) {
-                long sessionId = session.getSessionId();
-                attempts++;
-                if (!fileReferenceExistsOnDisk(downloadDirectory, appFileReference.get())) {
-                    log.fine(() -> "Downloading application package with file reference " + appFileReference +
-                            " for " + applicationId + " (session " + sessionId + ")");
-
-                    FileReferenceDownload download = new FileReferenceDownload(appFileReference.get(),
-                                                                               this.getClass().getSimpleName(),
-                                                                               false);
-                    if (fileDownloader.getFile(download).isEmpty()) {
-                        failures++;
-                        log.info("Downloading application package (" + appFileReference + ")" +
-                                         " for " + applicationId + " (session " + sessionId + ") unsuccessful. " +
-                                         "Can be ignored unless it happens many times over a long period of time, retries is expected");
-                        continue;
+                var applicationId = session.getApplicationId();
+                log.finest(() -> "Verifying application package for " + applicationId);
+
+                Optional<FileReference> appFileReference = session.getApplicationPackageReference();
+                if (appFileReference.isPresent()) {
+                    long sessionId = session.getSessionId();
+                    attempts++;
+                    if (!fileReferenceExistsOnDisk(downloadDirectory, appFileReference.get())) {
+                        log.fine(() -> "Downloading application package with file reference " + appFileReference +
+                                       " for " + applicationId + " (session " + sessionId + ")");
+
+                        FileReferenceDownload download = new FileReferenceDownload(appFileReference.get(),
+                                                                                   this.getClass().getSimpleName(),
+                                                                                   false);
+                        Future<Optional<File>> futureDownload = fileDownloader.getFutureFileOrTimeout(download);
+                        futureDownloads.add(() -> {
+                            try {
+                                if (futureDownload.get().isPresent()) {
+                                    createLocalSessionIfMissing(applicationId, sessionId);
+                                    return;
+                                }
+                            }
+                            catch (Exception ignored) { }
+                            failures[0]++;
+                            log.info("Downloading application package (" + appFileReference + ")" +
+                                     " for " + applicationId + " (session " + sessionId + ") unsuccessful. " +
+                                     "Can be ignored unless it happens many times over a long period of time, retries is expected");
+                        });
+                    }
+                    else {
+                        createLocalSessionIfMissing(applicationId, sessionId);
                     }
                 }
-                createLocalSessionIfMissing(applicationId, sessionId);
             }
-        }
-        return  asSuccessFactorDeviation(attempts, failures);
+
+        futureDownloads.forEach(Runnable::run);
+
+        return  asSuccessFactorDeviation(attempts, failures[0]);
     }
 
     private static FileDownloader createFileDownloader(ApplicationRepository applicationRepository,