From e9164ba1dda6e3051d3d329d9ae2508d295b4ad2 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 20 Oct 2021 11:02:50 +0200 Subject: Check if file exists before trying to download --- .../vespa/filedistribution/FileDownloader.java | 26 +++++++++++++++------- .../filedistribution/FileReferenceDownloader.java | 7 +++++- .../vespa/filedistribution/FileDownloaderTest.java | 1 + 3 files changed, 25 insertions(+), 9 deletions(-) (limited to 'filedistribution/src') diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 0678509dc68..13b6f5d8cc3 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -64,7 +64,11 @@ public class FileDownloader implements AutoCloseable { this.sleepBetweenRetries = sleepBetweenRetries; // Needed to receive RPC receiveFile* calls from server after starting download of file reference new FileReceiver(supervisor, downloads, downloadDirectory); - this.fileReferenceDownloader = new FileReferenceDownloader(connectionPool, downloads, timeout, sleepBetweenRetries); + this.fileReferenceDownloader = new FileReferenceDownloader(connectionPool, + downloads, + timeout, + sleepBetweenRetries, + downloadDirectory); } public Optional getFile(FileReference fileReference) { @@ -86,9 +90,12 @@ public class FileDownloader implements AutoCloseable { FileReference fileReference = fileReferenceDownload.fileReference(); Optional file = getFileFromFileSystem(fileReference); - return (file.isPresent()) - ? CompletableFuture.completedFuture(file) - : startDownload(fileReferenceDownload); + if (file.isPresent()) { + downloads.setDownloadStatus(fileReference, 1.0); + return CompletableFuture.completedFuture(file); + } else { + return startDownload(fileReferenceDownload); + } } public Map downloadStatus() { return downloads.downloadStatus(); } @@ -102,6 +109,10 @@ public class FileDownloader implements AutoCloseable { } private Optional getFileFromFileSystem(FileReference fileReference) { + return getFileFromFileSystem(fileReference, downloadDirectory); + } + + private static Optional getFileFromFileSystem(FileReference fileReference, File downloadDirectory) { File[] files = new File(downloadDirectory, fileReference.value()).listFiles(); if (files == null) return Optional.empty(); if (files.length == 0) return Optional.empty(); @@ -114,13 +125,12 @@ public class FileDownloader implements AutoCloseable { throw new RuntimeException("File reference '" + fileReference.value() + "' exists, but unable to read it"); } else { log.log(Level.FINE, () -> "File reference '" + fileReference.value() + "' found: " + file.getAbsolutePath()); - downloads.setDownloadStatus(fileReference, 1.0); return Optional.of(file); } } - boolean fileReferenceExists(FileReference fileReference) { - return getFileFromFileSystem(fileReference).isPresent(); + static boolean fileReferenceExists(FileReference fileReference, File downloadDirectory) { + return getFileFromFileSystem(fileReference, downloadDirectory).isPresent(); } boolean isDownloading(FileReference fileReference) { @@ -129,7 +139,7 @@ public class FileDownloader implements AutoCloseable { /** Start a download if needed, don't wait for result */ public void downloadIfNeeded(FileReferenceDownload fileReferenceDownload) { - if (fileReferenceExists(fileReferenceDownload.fileReference())) return; + if (fileReferenceExists(fileReferenceDownload.fileReference(), downloadDirectory)) return; startDownload(fileReferenceDownload); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 740bf23796f..445106f4fe4 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -36,15 +36,18 @@ public class FileReferenceDownloader { private final Duration downloadTimeout; private final Duration sleepBetweenRetries; private final Duration rpcTimeout; + private final File downloadDirectory; FileReferenceDownloader(ConnectionPool connectionPool, Downloads downloads, Duration timeout, - Duration sleepBetweenRetries) { + Duration sleepBetweenRetries, + File downloadDirectory) { this.connectionPool = connectionPool; this.downloads = downloads; this.downloadTimeout = timeout; this.sleepBetweenRetries = sleepBetweenRetries; + this.downloadDirectory = downloadDirectory; String timeoutString = System.getenv("VESPA_CONFIGPROXY_FILEDOWNLOAD_RPC_TIMEOUT"); this.rpcTimeout = Duration.ofSeconds(timeoutString == null ? 30 : Integer.parseInt(timeoutString)); } @@ -53,6 +56,8 @@ public class FileReferenceDownloader { FileReference fileReference = fileReferenceDownload.fileReference(); int retryCount = 0; do { + if (FileDownloader.fileReferenceExists(fileReference, downloadDirectory)) + return; if (startDownloadRpc(fileReferenceDownload, retryCount)) return; diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index 79530c39ad7..97b948ef5d4 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -76,6 +76,7 @@ public class FileDownloaderTest { FileReference fileReference = new FileReference(fileReferenceString); File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); writeFileReference(downloadDir, fileReferenceString, filename); + fileDownloader.downloads().completedDownloading(fileReference, fileReferenceFullPath); // Check that we get correct path and content when asking for file reference Optional pathToFile = fileDownloader.getFile(fileReference); -- cgit v1.2.3