diff options
author | Harald Musum <musum@verizonmedia.com> | 2021-06-14 20:07:06 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2021-06-14 20:07:06 +0200 |
commit | 85d3f65e2cdf0e7c9abc89ac64042ee1101d7e40 (patch) | |
tree | 0ed9d2c96f73a00bf29d5e6ec22cdaeb049827ed /filedistribution/src | |
parent | 0f52dbe1b874dc709cf0b4eacc9fa0cc4b14e942 (diff) |
Temp directory is always the same as download diredctory, simplify
Diffstat (limited to 'filedistribution/src')
4 files changed, 16 insertions, 25 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index e44a4619f70..9d2ee96340d 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -41,15 +41,15 @@ public class FileDownloader implements AutoCloseable { public FileDownloader(ConnectionPool connectionPool, File downloadDirectory, Downloads downloads) { // TODO: Reduce timeout even more, timeout is so long that we might get starvation - this(connectionPool, downloadDirectory, downloadDirectory, downloads, Duration.ofMinutes(5), Duration.ofSeconds(10)); + this(connectionPool, downloadDirectory, downloads, Duration.ofMinutes(5), Duration.ofSeconds(10)); } - public FileDownloader(ConnectionPool connectionPool, File downloadDirectory, File tmpDirectory, Downloads downloads, + public FileDownloader(ConnectionPool connectionPool, File downloadDirectory, Downloads downloads, Duration timeout, Duration sleepBetweenRetries) { this.downloadDirectory = downloadDirectory; this.timeout = timeout; // Needed to receive RPC calls receiveFile* from server after asking for files - new FileReceiver(connectionPool.getSupervisor(), downloads, downloadDirectory, tmpDirectory); + new FileReceiver(connectionPool.getSupervisor(), downloads, downloadDirectory); this.fileReferenceDownloader = new FileReferenceDownloader(connectionPool, downloads, timeout, sleepBetweenRetries); this.downloads = downloads; } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 875ca3cac29..e1a8cf92513 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -41,9 +41,6 @@ public class FileReceiver { private final Supervisor supervisor; private final Downloads downloads; private final File downloadDirectory; - // Should be on same partition as downloadDirectory to make sure moving files from tmpDirectory - // to downloadDirectory is atomic - private final File tmpDirectory; private final AtomicInteger nextSessionId = new AtomicInteger(1); private final Map<Integer, Session> sessions = new HashMap<>(); @@ -61,7 +58,7 @@ public class FileReceiver { private final File tmpDir; private final File inprogressFile; - Session(File downloadDirectory, File tmpDirectory, int sessionId, FileReference reference, + Session(File downloadDirectory, int sessionId, FileReference reference, FileReferenceData.Type fileType, String fileName, long fileSize) { this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0); @@ -74,12 +71,12 @@ public class FileReceiver { currentPartId = 0; currentHash = 0; fileReferenceDir = new File(downloadDirectory, reference.value()); - this.tmpDir = tmpDirectory; + this.tmpDir = downloadDirectory; try { - inprogressFile = Files.createTempFile(tmpDirectory.toPath(), fileName, ".inprogress").toFile(); + inprogressFile = Files.createTempFile(tmpDir.toPath(), fileName, ".inprogress").toFile(); } catch (IOException e) { - String msg = "Failed creating temp file for inprogress file for " + fileName + " in '" + tmpDirectory.toPath() + "': "; + String msg = "Failed creating temp file for inprogress file for " + fileName + " in '" + tmpDir.toPath() + "': "; log.log(Level.SEVERE, msg + e.getMessage(), e); throw new RuntimeException(msg, e); } @@ -149,11 +146,10 @@ public class FileReceiver { } } - FileReceiver(Supervisor supervisor, Downloads downloads, File downloadDirectory, File tmpDirectory) { + FileReceiver(Supervisor supervisor, Downloads downloads, File downloadDirectory) { this.supervisor = supervisor; this.downloads = downloads; this.downloadDirectory = downloadDirectory; - this.tmpDirectory = tmpDirectory; registerMethods(); } @@ -231,7 +227,7 @@ public class FileReceiver { log.severe("Session id " + sessionId + " already exist, impossible. Request from(" + req.target() + ")"); } else { try { - sessions.put(sessionId, new Session(downloadDirectory, tmpDirectory, sessionId, reference, + sessions.put(sessionId, new Session(downloadDirectory, sessionId, reference, FileReferenceData.Type.valueOf(type),fileName, fileSize)); } catch (Exception e) { retval = 1; diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index 875159be8f9..1d11b358540 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -44,17 +44,14 @@ public class FileDownloaderTest { private Downloads downloads; private FileDownloader fileDownloader; private File downloadDir; - private File tempDir; @Before public void setup() { try { downloadDir = Files.createTempDirectory("filedistribution").toFile(); - tempDir = Files.createTempDirectory("download").toFile(); connection = new MockConnection(); downloads = new Downloads(); - fileDownloader = new FileDownloader(connection, downloadDir, tempDir, downloads, - Duration.ofSeconds(1), sleepBetweenRetries); + fileDownloader = new FileDownloader(connection, downloadDir, downloads, Duration.ofSeconds(1), sleepBetweenRetries); } catch (IOException e) { e.printStackTrace(); fail(e.getMessage()); @@ -165,7 +162,7 @@ public class FileDownloaderTest { @Test public void getFileWhenConnectionError() throws IOException { - fileDownloader = new FileDownloader(connection, downloadDir, tempDir, downloads, Duration.ofSeconds(2), sleepBetweenRetries); + fileDownloader = new FileDownloader(connection, downloadDir, downloads, Duration.ofSeconds(2), sleepBetweenRetries); File downloadDir = fileDownloader.downloadDirectory(); int timesToFail = 2; @@ -199,7 +196,7 @@ public class FileDownloaderTest { public void getFileWhenDownloadInProgress() throws IOException, ExecutionException, InterruptedException { ExecutorService executor = Executors.newFixedThreadPool(2); String filename = "abc.jar"; - fileDownloader = new FileDownloader(connection, downloadDir, tempDir, downloads, Duration.ofSeconds(3), sleepBetweenRetries); + fileDownloader = new FileDownloader(connection, downloadDir, downloads, Duration.ofSeconds(3), sleepBetweenRetries); File downloadDir = fileDownloader.downloadDirectory(); // Delay response so that we can make a second request while downloading the file from the first request @@ -239,7 +236,7 @@ public class FileDownloaderTest { Duration timeout = Duration.ofMillis(200); MockConnection connectionPool = new MockConnection(); connectionPool.setResponseHandler(new MockConnection.WaitResponseHandler(timeout.plus(Duration.ofMillis(1000)))); - FileDownloader fileDownloader = new FileDownloader(connectionPool, downloadDir, tempDir, downloads, timeout, sleepBetweenRetries); + FileDownloader fileDownloader = new FileDownloader(connectionPool, downloadDir, downloads, timeout, sleepBetweenRetries); FileReference foo = new FileReference("foo"); // Should download since we do not have the file on disk fileDownloader.downloadIfNeeded(new FileReferenceDownload(foo)); @@ -287,7 +284,7 @@ public class FileDownloaderTest { FileReferenceData.Type type, byte[] content) { XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); FileReceiver.Session session = - new FileReceiver.Session(downloadDir, tempDir, 1, fileReference, type, filename, content.length); + new FileReceiver.Session(downloadDir, 1, fileReference, type, filename, content.length); session.addPart(0, content); File file = session.close(hasher.hash(ByteBuffer.wrap(content), 0)); downloads.completedDownloading(fileReference, file); diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java index a9ddff655e3..69d4344d246 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java @@ -21,7 +21,6 @@ import java.nio.file.Files; public class FileReceiverTest { private File root; - private File tempDir; private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); @Rule @@ -30,7 +29,6 @@ public class FileReceiverTest { @Before public void setup() throws IOException { root = temporaryFolder.newFolder("root"); - tempDir = temporaryFolder.newFolder("tmp"); } @Test @@ -70,7 +68,7 @@ public class FileReceiverTest { private void transferPartsAndAssert(FileReference ref, String fileName, String all, int numParts) throws IOException { byte [] allContent = Utf8.toBytes(all); - FileReceiver.Session session = new FileReceiver.Session(root, tempDir, 1, ref, + FileReceiver.Session session = new FileReceiver.Session(root, 1, ref, FileReferenceData.Type.file, fileName, allContent.length); int partSize = (allContent.length+(numParts-1))/numParts; ByteBuffer bb = ByteBuffer.wrap(allContent); @@ -91,7 +89,7 @@ public class FileReceiverTest { private void transferCompressedData(FileReference ref, String fileName, byte[] data) { FileReceiver.Session session = - new FileReceiver.Session(root, tempDir, 1, ref, FileReferenceData.Type.compressed, fileName, data.length); + new FileReceiver.Session(root, 1, ref, FileReferenceData.Type.compressed, fileName, data.length); session.addPart(0, data); session.close(hasher.hash(ByteBuffer.wrap(data), 0)); } |