summaryrefslogtreecommitdiffstats
path: root/filedistribution
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2021-06-14 20:07:06 +0200
committerHarald Musum <musum@verizonmedia.com>2021-06-14 20:07:06 +0200
commit85d3f65e2cdf0e7c9abc89ac64042ee1101d7e40 (patch)
tree0ed9d2c96f73a00bf29d5e6ec22cdaeb049827ed /filedistribution
parent0f52dbe1b874dc709cf0b4eacc9fa0cc4b14e942 (diff)
Temp directory is always the same as download diredctory, simplify
Diffstat (limited to 'filedistribution')
-rw-r--r--filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java6
-rw-r--r--filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java16
-rw-r--r--filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java13
-rw-r--r--filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java6
4 files changed, 16 insertions, 25 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
index e44a4619f70..9d2ee96340d 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
@@ -41,15 +41,15 @@ public class FileDownloader implements AutoCloseable {
public FileDownloader(ConnectionPool connectionPool, File downloadDirectory, Downloads downloads) {
// TODO: Reduce timeout even more, timeout is so long that we might get starvation
- this(connectionPool, downloadDirectory, downloadDirectory, downloads, Duration.ofMinutes(5), Duration.ofSeconds(10));
+ this(connectionPool, downloadDirectory, downloads, Duration.ofMinutes(5), Duration.ofSeconds(10));
}
- public FileDownloader(ConnectionPool connectionPool, File downloadDirectory, File tmpDirectory, Downloads downloads,
+ public FileDownloader(ConnectionPool connectionPool, File downloadDirectory, Downloads downloads,
Duration timeout, Duration sleepBetweenRetries) {
this.downloadDirectory = downloadDirectory;
this.timeout = timeout;
// Needed to receive RPC calls receiveFile* from server after asking for files
- new FileReceiver(connectionPool.getSupervisor(), downloads, downloadDirectory, tmpDirectory);
+ new FileReceiver(connectionPool.getSupervisor(), downloads, downloadDirectory);
this.fileReferenceDownloader = new FileReferenceDownloader(connectionPool, downloads, timeout, sleepBetweenRetries);
this.downloads = downloads;
}
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
index 875ca3cac29..e1a8cf92513 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
@@ -41,9 +41,6 @@ public class FileReceiver {
private final Supervisor supervisor;
private final Downloads downloads;
private final File downloadDirectory;
- // Should be on same partition as downloadDirectory to make sure moving files from tmpDirectory
- // to downloadDirectory is atomic
- private final File tmpDirectory;
private final AtomicInteger nextSessionId = new AtomicInteger(1);
private final Map<Integer, Session> sessions = new HashMap<>();
@@ -61,7 +58,7 @@ public class FileReceiver {
private final File tmpDir;
private final File inprogressFile;
- Session(File downloadDirectory, File tmpDirectory, int sessionId, FileReference reference,
+ Session(File downloadDirectory, int sessionId, FileReference reference,
FileReferenceData.Type fileType, String fileName, long fileSize)
{
this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0);
@@ -74,12 +71,12 @@ public class FileReceiver {
currentPartId = 0;
currentHash = 0;
fileReferenceDir = new File(downloadDirectory, reference.value());
- this.tmpDir = tmpDirectory;
+ this.tmpDir = downloadDirectory;
try {
- inprogressFile = Files.createTempFile(tmpDirectory.toPath(), fileName, ".inprogress").toFile();
+ inprogressFile = Files.createTempFile(tmpDir.toPath(), fileName, ".inprogress").toFile();
} catch (IOException e) {
- String msg = "Failed creating temp file for inprogress file for " + fileName + " in '" + tmpDirectory.toPath() + "': ";
+ String msg = "Failed creating temp file for inprogress file for " + fileName + " in '" + tmpDir.toPath() + "': ";
log.log(Level.SEVERE, msg + e.getMessage(), e);
throw new RuntimeException(msg, e);
}
@@ -149,11 +146,10 @@ public class FileReceiver {
}
}
- FileReceiver(Supervisor supervisor, Downloads downloads, File downloadDirectory, File tmpDirectory) {
+ FileReceiver(Supervisor supervisor, Downloads downloads, File downloadDirectory) {
this.supervisor = supervisor;
this.downloads = downloads;
this.downloadDirectory = downloadDirectory;
- this.tmpDirectory = tmpDirectory;
registerMethods();
}
@@ -231,7 +227,7 @@ public class FileReceiver {
log.severe("Session id " + sessionId + " already exist, impossible. Request from(" + req.target() + ")");
} else {
try {
- sessions.put(sessionId, new Session(downloadDirectory, tmpDirectory, sessionId, reference,
+ sessions.put(sessionId, new Session(downloadDirectory, sessionId, reference,
FileReferenceData.Type.valueOf(type),fileName, fileSize));
} catch (Exception e) {
retval = 1;
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
index 875159be8f9..1d11b358540 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
@@ -44,17 +44,14 @@ public class FileDownloaderTest {
private Downloads downloads;
private FileDownloader fileDownloader;
private File downloadDir;
- private File tempDir;
@Before
public void setup() {
try {
downloadDir = Files.createTempDirectory("filedistribution").toFile();
- tempDir = Files.createTempDirectory("download").toFile();
connection = new MockConnection();
downloads = new Downloads();
- fileDownloader = new FileDownloader(connection, downloadDir, tempDir, downloads,
- Duration.ofSeconds(1), sleepBetweenRetries);
+ fileDownloader = new FileDownloader(connection, downloadDir, downloads, Duration.ofSeconds(1), sleepBetweenRetries);
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
@@ -165,7 +162,7 @@ public class FileDownloaderTest {
@Test
public void getFileWhenConnectionError() throws IOException {
- fileDownloader = new FileDownloader(connection, downloadDir, tempDir, downloads, Duration.ofSeconds(2), sleepBetweenRetries);
+ fileDownloader = new FileDownloader(connection, downloadDir, downloads, Duration.ofSeconds(2), sleepBetweenRetries);
File downloadDir = fileDownloader.downloadDirectory();
int timesToFail = 2;
@@ -199,7 +196,7 @@ public class FileDownloaderTest {
public void getFileWhenDownloadInProgress() throws IOException, ExecutionException, InterruptedException {
ExecutorService executor = Executors.newFixedThreadPool(2);
String filename = "abc.jar";
- fileDownloader = new FileDownloader(connection, downloadDir, tempDir, downloads, Duration.ofSeconds(3), sleepBetweenRetries);
+ fileDownloader = new FileDownloader(connection, downloadDir, downloads, Duration.ofSeconds(3), sleepBetweenRetries);
File downloadDir = fileDownloader.downloadDirectory();
// Delay response so that we can make a second request while downloading the file from the first request
@@ -239,7 +236,7 @@ public class FileDownloaderTest {
Duration timeout = Duration.ofMillis(200);
MockConnection connectionPool = new MockConnection();
connectionPool.setResponseHandler(new MockConnection.WaitResponseHandler(timeout.plus(Duration.ofMillis(1000))));
- FileDownloader fileDownloader = new FileDownloader(connectionPool, downloadDir, tempDir, downloads, timeout, sleepBetweenRetries);
+ FileDownloader fileDownloader = new FileDownloader(connectionPool, downloadDir, downloads, timeout, sleepBetweenRetries);
FileReference foo = new FileReference("foo");
// Should download since we do not have the file on disk
fileDownloader.downloadIfNeeded(new FileReferenceDownload(foo));
@@ -287,7 +284,7 @@ public class FileDownloaderTest {
FileReferenceData.Type type, byte[] content) {
XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
FileReceiver.Session session =
- new FileReceiver.Session(downloadDir, tempDir, 1, fileReference, type, filename, content.length);
+ new FileReceiver.Session(downloadDir, 1, fileReference, type, filename, content.length);
session.addPart(0, content);
File file = session.close(hasher.hash(ByteBuffer.wrap(content), 0));
downloads.completedDownloading(fileReference, file);
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
index a9ddff655e3..69d4344d246 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
@@ -21,7 +21,6 @@ import java.nio.file.Files;
public class FileReceiverTest {
private File root;
- private File tempDir;
private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
@Rule
@@ -30,7 +29,6 @@ public class FileReceiverTest {
@Before
public void setup() throws IOException {
root = temporaryFolder.newFolder("root");
- tempDir = temporaryFolder.newFolder("tmp");
}
@Test
@@ -70,7 +68,7 @@ public class FileReceiverTest {
private void transferPartsAndAssert(FileReference ref, String fileName, String all, int numParts) throws IOException {
byte [] allContent = Utf8.toBytes(all);
- FileReceiver.Session session = new FileReceiver.Session(root, tempDir, 1, ref,
+ FileReceiver.Session session = new FileReceiver.Session(root, 1, ref,
FileReferenceData.Type.file, fileName, allContent.length);
int partSize = (allContent.length+(numParts-1))/numParts;
ByteBuffer bb = ByteBuffer.wrap(allContent);
@@ -91,7 +89,7 @@ public class FileReceiverTest {
private void transferCompressedData(FileReference ref, String fileName, byte[] data) {
FileReceiver.Session session =
- new FileReceiver.Session(root, tempDir, 1, ref, FileReferenceData.Type.compressed, fileName, data.length);
+ new FileReceiver.Session(root, 1, ref, FileReferenceData.Type.compressed, fileName, data.length);
session.addPart(0, data);
session.close(hasher.hash(ByteBuffer.wrap(data), 0));
}