Check if file exists before trying to download

3 files changed, 25 insertions, 9 deletions

diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
index 0678509dc68..13b6f5d8cc3 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
@@ -64,7 +64,11 @@ public class FileDownloader implements AutoCloseable {
         this.sleepBetweenRetries = sleepBetweenRetries;
         // Needed to receive RPC receiveFile* calls from server after starting download of file reference
         new FileReceiver(supervisor, downloads, downloadDirectory);
-        this.fileReferenceDownloader = new FileReferenceDownloader(connectionPool, downloads, timeout, sleepBetweenRetries);
+        this.fileReferenceDownloader = new FileReferenceDownloader(connectionPool,
+                                                                   downloads,
+                                                                   timeout,
+                                                                   sleepBetweenRetries,
+                                                                   downloadDirectory);
     }
 
     public Optional<File> getFile(FileReference fileReference) {
@@ -86,9 +90,12 @@ public class FileDownloader implements AutoCloseable {
         FileReference fileReference = fileReferenceDownload.fileReference();
 
         Optional<File> file = getFileFromFileSystem(fileReference);
-        return (file.isPresent())
-                ? CompletableFuture.completedFuture(file)
-                : startDownload(fileReferenceDownload);
+        if (file.isPresent()) {
+            downloads.setDownloadStatus(fileReference, 1.0);
+            return CompletableFuture.completedFuture(file);
+        } else {
+            return startDownload(fileReferenceDownload);
+        }
     }
 
     public Map<FileReference, Double> downloadStatus() { return downloads.downloadStatus(); }
@@ -102,6 +109,10 @@ public class FileDownloader implements AutoCloseable {
     }
 
     private Optional<File> getFileFromFileSystem(FileReference fileReference) {
+        return getFileFromFileSystem(fileReference, downloadDirectory);
+    }
+
+    private static Optional<File> getFileFromFileSystem(FileReference fileReference, File downloadDirectory) {
         File[] files = new File(downloadDirectory, fileReference.value()).listFiles();
         if (files == null) return Optional.empty();
         if (files.length == 0) return Optional.empty();
@@ -114,13 +125,12 @@ public class FileDownloader implements AutoCloseable {
             throw new RuntimeException("File reference '" + fileReference.value() + "' exists, but unable to read it");
         } else {
             log.log(Level.FINE, () -> "File reference '" + fileReference.value() + "' found: " + file.getAbsolutePath());
-            downloads.setDownloadStatus(fileReference, 1.0);
             return Optional.of(file);
         }
     }
 
-    boolean fileReferenceExists(FileReference fileReference) {
-        return getFileFromFileSystem(fileReference).isPresent();
+    static boolean fileReferenceExists(FileReference fileReference, File downloadDirectory) {
+        return getFileFromFileSystem(fileReference, downloadDirectory).isPresent();
     }
 
     boolean isDownloading(FileReference fileReference) {
@@ -129,7 +139,7 @@ public class FileDownloader implements AutoCloseable {
 
     /** Start a download if needed, don't wait for result */
     public void downloadIfNeeded(FileReferenceDownload fileReferenceDownload) {
-        if (fileReferenceExists(fileReferenceDownload.fileReference())) return;
+        if (fileReferenceExists(fileReferenceDownload.fileReference(), downloadDirectory)) return;
 
         startDownload(fileReferenceDownload);
     }
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
index 740bf23796f..445106f4fe4 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
@@ -36,15 +36,18 @@ public class FileReferenceDownloader {
     private final Duration downloadTimeout;
     private final Duration sleepBetweenRetries;
     private final Duration rpcTimeout;
+    private final File downloadDirectory;
 
     FileReferenceDownloader(ConnectionPool connectionPool,
                             Downloads downloads,
                             Duration timeout,
-                            Duration sleepBetweenRetries) {
+                            Duration sleepBetweenRetries,
+                            File downloadDirectory) {
         this.connectionPool = connectionPool;
         this.downloads = downloads;
         this.downloadTimeout = timeout;
         this.sleepBetweenRetries = sleepBetweenRetries;
+        this.downloadDirectory = downloadDirectory;
         String timeoutString = System.getenv("VESPA_CONFIGPROXY_FILEDOWNLOAD_RPC_TIMEOUT");
         this.rpcTimeout = Duration.ofSeconds(timeoutString == null ? 30 : Integer.parseInt(timeoutString));
     }
@@ -53,6 +56,8 @@ public class FileReferenceDownloader {
         FileReference fileReference = fileReferenceDownload.fileReference();
         int retryCount = 0;
         do {
+            if (FileDownloader.fileReferenceExists(fileReference, downloadDirectory))
+                return;
             if (startDownloadRpc(fileReferenceDownload, retryCount))
                 return;
 
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
index 79530c39ad7..97b948ef5d4 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
@@ -76,6 +76,7 @@ public class FileDownloaderTest {
             FileReference fileReference = new FileReference(fileReferenceString);
             File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference);
             writeFileReference(downloadDir, fileReferenceString, filename);
+            fileDownloader.downloads().completedDownloading(fileReference, fileReferenceFullPath);
 
             // Check that we get correct path and content when asking for file reference
             Optional<File> pathToFile = fileDownloader.getFile(fileReference);