From 5124bc3a959a399483db3e52e721e789ba426338 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 12 Dec 2017 12:32:01 +0100 Subject: Make sure to use a tmp directory on same partition as download directory --- .../vespa/filedistribution/FileDownloader.java | 10 +++++----- .../yahoo/vespa/filedistribution/FileReceiver.java | 22 +++++++++++++++------- .../filedistribution/FileReferenceDownloader.java | 4 ++-- .../vespa/filedistribution/FileDownloaderTest.java | 8 +++++--- .../vespa/filedistribution/FileReceiverTest.java | 19 +++++++++++++++++-- 5 files changed, 44 insertions(+), 19 deletions(-) diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 32ae724cf0f..6610893d8ae 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -35,14 +35,15 @@ public class FileDownloader { public FileDownloader(ConnectionPool connectionPool) { this(connectionPool, - new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution")), - Duration.ofMinutes(15)); + new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution")), + new File(Defaults.getDefaults().underVespaHome("tmp")), + Duration.ofMinutes(15)); } - FileDownloader(ConnectionPool connectionPool, File downloadDirectory, Duration timeout) { + FileDownloader(ConnectionPool connectionPool, File downloadDirectory, File tmpDirectory, Duration timeout) { this.downloadDirectory = downloadDirectory; this.timeout = timeout; - this.fileReferenceDownloader = new FileReferenceDownloader(downloadDirectory, connectionPool, timeout); + this.fileReferenceDownloader = new FileReferenceDownloader(downloadDirectory, tmpDirectory, connectionPool, timeout); } public Optional getFile(FileReference fileReference) { @@ -67,7 +68,6 @@ public class FileDownloader { log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found in " + directory.getAbsolutePath() + ", starting download"); return queueForDownload(fileReference, timeout); - } } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 73ca8de1e9c..c4e010e57d5 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -34,6 +34,9 @@ public class FileReceiver { private final Supervisor supervisor; private final FileReferenceDownloader downloader; private final File downloadDirectory; + // Should be on same partition as downloadDirectory to make sure moving files from tmpDirectory + // to downloadDirectory is atomic + private final File tmpDirectory; private final AtomicInteger nextSessionId = new AtomicInteger(1); private final Map sessions = new HashMap<>(); @@ -48,9 +51,10 @@ public class FileReceiver { private long currentPartId; private long currentHash; private final File fileReferenceDir; + private final File tmpDir; private final File inprogressFile; - Session(File downloadDirectory, int sessionId, FileReference reference, + Session(File downloadDirectory, File tmpDirectory, int sessionId, FileReference reference, FileReferenceData.Type fileType, String fileName, long fileSize) { this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0); @@ -63,6 +67,7 @@ public class FileReceiver { currentPartId = 0; currentHash = 0; fileReferenceDir = new File(downloadDirectory, reference.value()); + this.tmpDir = tmpDirectory; try { Files.createDirectories(fileReferenceDir.toPath()); } catch (IOException e) { @@ -71,7 +76,7 @@ public class FileReceiver { } try { - inprogressFile = Files.createTempFile(downloadDirectory.toPath(), fileName, ".inprogress").toFile(); + inprogressFile = Files.createTempFile(tmpDirectory.toPath(), fileName, ".inprogress").toFile(); } catch (IOException e) { String msg = "Failed creating tempfile for inprogress file for(" + fileName + ") in '" + fileReferenceDir.toPath() + "': "; log.log(LogLevel.ERROR, msg + e.getMessage(), e); @@ -97,6 +102,7 @@ public class FileReceiver { currentPartId++; hasher.update(part, 0, part.length); } + File close(long hash) { if (hasher.getValue() != hash) { throw new RuntimeException("xxhash from content (" + currentHash + ") is not equal to xxhash in request (" + hash + ")"); @@ -105,7 +111,7 @@ public class FileReceiver { try { // Unpack if necessary if (fileType == FileReferenceData.Type.compressed) { - File decompressedDir = Files.createTempDirectory("archive").toFile(); + File decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); log.log(LogLevel.DEBUG, "Archived file, unpacking " + inprogressFile + " to " + decompressedDir); CompressedFileReference.decompress(inprogressFile, decompressedDir); moveFileToDestination(decompressedDir, fileReferenceDir); @@ -121,10 +127,11 @@ public class FileReceiver { } } - FileReceiver(Supervisor supervisor, FileReferenceDownloader downloader, File downloadDirectory) { + FileReceiver(Supervisor supervisor, FileReferenceDownloader downloader, File downloadDirectory, File tmpDirectory) { this.supervisor = supervisor; this.downloader = downloader; this.downloadDirectory = downloadDirectory; + this.tmpDirectory = tmpDirectory; registerMethods(); } @@ -201,12 +208,13 @@ public class FileReceiver { // file might be a directory (and then type is compressed) File file = new File(fileReferenceDir, fileReferenceData.filename()); try { - File tempFile = new File(Files.createTempDirectory("downloaded").toFile(), fileReferenceData.filename()); + File tempDownloadedDir = Files.createTempDirectory(tmpDirectory.toPath(), "downloaded").toFile(); + File tempFile = new File(tempDownloadedDir, fileReferenceData.filename()); Files.write(tempFile.toPath(), fileReferenceData.content().array()); // Unpack if necessary if (fileReferenceData.type() == FileReferenceData.Type.compressed) { - File decompressedDir = Files.createTempDirectory("decompressed").toFile(); + File decompressedDir = Files.createTempDirectory(tempDownloadedDir.toPath(), "decompressed").toFile(); log.log(LogLevel.DEBUG, "Compressed file, unpacking " + tempFile + " to " + decompressedDir); CompressedFileReference.decompress(tempFile, decompressedDir); moveFileToDestination(decompressedDir, fileReferenceDir); @@ -252,7 +260,7 @@ public class FileReceiver { log.severe("Session id " + sessionId + " already exist, impossible. Request from(" + req.target() + ")"); } else { try { - sessions.put(sessionId, new Session(downloadDirectory, sessionId, reference, + sessions.put(sessionId, new Session(downloadDirectory, tmpDirectory, sessionId, reference, FileReferenceData.Type.valueOf(type),fileName, fileSize)); } catch (Exception e) { retval = 1; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 60919ecb0cf..7793e25f7b2 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -44,10 +44,10 @@ public class FileReferenceDownloader { private final Duration downloadTimeout; private final FileReceiver fileReceiver; - FileReferenceDownloader(File downloadDirectory, ConnectionPool connectionPool, Duration timeout) { + FileReferenceDownloader(File downloadDirectory, File tmpDirectory, ConnectionPool connectionPool, Duration timeout) { this.connectionPool = connectionPool; this.downloadTimeout = timeout; - this.fileReceiver = new FileReceiver(connectionPool.getSupervisor(), this, downloadDirectory); + this.fileReceiver = new FileReceiver(connectionPool.getSupervisor(), this, downloadDirectory, tmpDirectory); } private void startDownload(FileReference fileReference, Duration timeout, diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index 4618b229de1..60478550084 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -36,13 +36,15 @@ public class FileDownloaderTest { private MockConnection connection; private FileDownloader fileDownloader; private File downloadDir; + private File tempDir; @Before public void setup() { try { downloadDir = Files.createTempDirectory("filedistribution").toFile(); + tempDir = Files.createTempDirectory("download").toFile(); connection = new MockConnection(); - fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(2000)); + fileDownloader = new FileDownloader(connection, downloadDir, tempDir, Duration.ofMillis(2000)); } catch (IOException e) { e.printStackTrace(); fail(e.getMessage()); @@ -147,7 +149,7 @@ public class FileDownloaderTest { @Test public void getFileWhenConnectionError() throws IOException { - fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000)); + fileDownloader = new FileDownloader(connection, downloadDir, tempDir, Duration.ofMillis(3000)); File downloadDir = fileDownloader.downloadDirectory(); int timesToFail = 2; @@ -183,7 +185,7 @@ public class FileDownloaderTest { File downloadDir = Files.createTempDirectory("filedistribution").toFile(); MockConnection connectionPool = new MockConnection(); connectionPool.setResponseHandler(new MockConnection.WaitResponseHandler(timeout.plus(Duration.ofMillis(1000)))); - FileDownloader fileDownloader = new FileDownloader(connectionPool, downloadDir, timeout); + FileDownloader fileDownloader = new FileDownloader(connectionPool, downloadDir, tempDir, timeout); FileReference foo = new FileReference("foo"); FileReference bar = new FileReference("bar"); List fileReferences = Arrays.asList(foo, bar); diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java index 5edd1151cb1..34ce53ad4c8 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java @@ -4,7 +4,11 @@ import com.yahoo.config.FileReference; import com.yahoo.text.Utf8; import net.jpountz.xxhash.XXHash64; import net.jpountz.xxhash.XXHashFactory; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + import static org.junit.Assert.assertEquals; @@ -14,8 +18,19 @@ import java.nio.ByteBuffer; import java.nio.file.Files; public class FileReceiverTest { - private final File root = new File("."); + private File root; + private File tempDir; private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Before + public void setup() throws IOException { + root = temporaryFolder.newFolder("root"); + tempDir = temporaryFolder.newFolder("tmp"); + } + @Test public void receiveMultiPartFile() throws IOException{ @@ -40,7 +55,7 @@ public class FileReceiverTest { private String transferParts(FileReference ref, String fileName, String all, int numParts) throws IOException { byte [] allContent = Utf8.toBytes(all); - FileReceiver.Session session = new FileReceiver.Session(root, 1, ref, + FileReceiver.Session session = new FileReceiver.Session(root, tempDir, 1, ref, FileReferenceData.Type.file, fileName, allContent.length); int partSize = (allContent.length+(numParts-1))/numParts; ByteBuffer bb = ByteBuffer.wrap(allContent); -- cgit v1.2.3