diff options
author | Harald Musum <musum@oath.com> | 2017-12-07 13:18:16 +0100 |
---|---|---|
committer | Harald Musum <musum@oath.com> | 2017-12-07 13:18:16 +0100 |
commit | 423a955e4a975d4f1dde550df2cea018882d4035 (patch) | |
tree | d73dcbd41b924c20e467f5983f42cd397567074e /filedistribution/src/test | |
parent | 56deb93f655849712b4cc17fc69df6331e0253a3 (diff) |
Handle that a file reference is a directory with many files
Compress on the fly if asked for such a file reference
Diffstat (limited to 'filedistribution/src/test')
-rw-r--r-- | filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java | 56 |
1 files changed, 44 insertions, 12 deletions
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index 278c46dab8b..1c9e8cdb91b 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -13,15 +13,13 @@ import com.yahoo.jrt.Transport; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; -import net.jpountz.xxhash.XXHash64; -import net.jpountz.xxhash.XXHashFactory; import org.junit.Before; import org.junit.Test; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; import java.util.Arrays; import java.util.List; @@ -35,8 +33,6 @@ import static org.junit.Assert.fail; public class FileDownloaderTest { - private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); - private MockConnection connection; private FileDownloader fileDownloader; private File downloadDir; @@ -102,7 +98,7 @@ public class FileDownloaderTest { // Receives fileReference, should return and make it available to caller String filename = "abc.jar"; - receiveFile(fileReference, filename, "some other content"); + receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content"); Optional<File> downloadedFile = fileDownloader.getFile(fileReference); assertTrue(downloadedFile.isPresent()); @@ -113,6 +109,40 @@ public class FileDownloaderTest { // Verify download status when downloaded assertDownloadStatus(fileDownloader, fileReference, 100.0); } + + { + // fileReference does not exist on disk, needs to be downloaded, is compressed data + + FileReference fileReference = new FileReference("fileReferenceToDirWithManyFiles"); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); + assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + + // Verify download status + assertDownloadStatus(fileDownloader, fileReference, 0.0); + + // Receives fileReference, should return and make it available to caller + String filename = "abc.tar.gz"; + Path tempPath = Files.createTempDirectory("dir"); + File subdir = new File(tempPath.toFile(), "subdir"); + File fooFile = new File(subdir, "foo"); + IOUtils.writeFile(fooFile, "foo", false); + File barFile = new File(subdir, "bar"); + IOUtils.writeFile(barFile, "bar", false); + + File tarFile = CompressedFileReference.compress(tempPath.toFile(), Arrays.asList(fooFile, barFile), new File(tempPath.toFile(), filename)); + byte[] tarredContent = IOUtils.readFileBytes(tarFile); + receiveFile(fileReference, filename, FileReferenceData.Type.compressed, tarredContent); + Optional<File> downloadedFile = fileDownloader.getFile(fileReference); + + assertTrue(downloadedFile.isPresent()); + File downloadedFoo = new File(fileReferenceFullPath, tempPath.relativize(fooFile.toPath()).toString()); + File downloadedBar = new File(fileReferenceFullPath, tempPath.relativize(barFile.toPath()).toString()); + assertEquals("foo", IOUtils.readFile(downloadedFoo)); + assertEquals("bar", IOUtils.readFile(downloadedBar)); + + // Verify download status when downloaded + assertDownloadStatus(fileDownloader, fileReference, 100.0); + } } @Test @@ -133,7 +163,7 @@ public class FileDownloaderTest { // Receives fileReference, should return and make it available to caller String filename = "abc.jar"; - receiveFile(fileReference, filename, "some other content"); + receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content"); Optional<File> downloadedFile = fileDownloader.getFile(fileReference); assertTrue(downloadedFile.isPresent()); @@ -168,7 +198,7 @@ public class FileDownloaderTest { public void receiveFile() throws IOException { FileReference foo = new FileReference("foo"); String filename = "foo.jar"; - receiveFile(foo, filename, "content"); + receiveFile(foo, filename, FileReferenceData.Type.file, "content"); File downloadedFile = new File(fileReferenceFullPath(downloadDir, foo), filename); assertEquals("content", IOUtils.readFile(downloadedFile)); } @@ -187,10 +217,12 @@ public class FileDownloaderTest { assertEquals(expectedDownloadStatus, downloadStatus, 0.0001); } - private void receiveFile(FileReference fileReference, String filename, String content) { - byte[] contentBytes = Utf8.toBytes(content); - long xxHashFromContent = hasher.hash(ByteBuffer.wrap(contentBytes), 0); - fileDownloader.receiveFile(fileReference, filename, contentBytes, xxHashFromContent); + private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, String content) { + receiveFile(fileReference, filename, type, Utf8.toBytes(content)); + } + + private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, byte[] content) { + fileDownloader.receiveFile(new FileReferenceData(fileReference, filename, type, content)); } private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { |