diff options
author | Harald Musum <musum@yahoo-inc.com> | 2017-11-16 18:05:30 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-11-16 18:05:30 +0100 |
commit | b07e68d6e9a0220136e0f33506ffc37864ed0a3b (patch) | |
tree | 6094468ea5fc9382bf940c74b98e9e5ee3ef1836 /filedistribution/src/test | |
parent | 39416a17871a5915fe9533a8d3dae3818dd6e10d (diff) |
Revert "Revert "Hmusum/move new filedistribution classes""
Diffstat (limited to 'filedistribution/src/test')
-rw-r--r-- | filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java | 279 |
1 files changed, 279 insertions, 0 deletions
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java new file mode 100644 index 00000000000..738b0888956 --- /dev/null +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -0,0 +1,279 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.io.IOUtils; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.RequestWaiter; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.Connection; +import com.yahoo.vespa.config.ConnectionPool; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FileDownloaderTest { + + private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + + private MockConnection connection; + private FileDownloader fileDownloader; + private File downloadDir; + + @Before + public void setup() { + try { + downloadDir = Files.createTempDirectory("filedistribution").toFile(); + connection = new MockConnection(); + fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000)); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void getFile() throws IOException { + File downloadDir = fileDownloader.downloadDirectory(); + + { + // fileReference already exists on disk, does not have to be downloaded + + String fileReferenceString = "foo"; + String filename = "foo.jar"; + FileReference fileReference = new FileReference(fileReferenceString); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); + writeFileReference(downloadDir, fileReferenceString, filename); + + // Check that we get correct path and content when asking for file reference + Optional<File> pathToFile = fileDownloader.getFile(fileReference); + assertTrue(pathToFile.isPresent()); + String downloadedFile = new File(fileReferenceFullPath, filename).getAbsolutePath(); + assertEquals(new File(fileReferenceFullPath, filename).getAbsolutePath(), downloadedFile); + assertEquals("content", IOUtils.readFile(pathToFile.get())); + + // Verify download status when downloaded + assertDownloadStatus(fileDownloader, fileReference, 100.0); + } + + { + // fileReference does not exist on disk, needs to be downloaded, but fails when asking upstream for file) + + connection.setResponseHandler(new MockConnection.UnknownFileReferenceResponseHandler()); + + FileReference fileReference = new FileReference("bar"); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); + assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + + // Verify download status when unable to download + assertDownloadStatus(fileDownloader, fileReference, 0.0); + } + + { + // fileReference does not exist on disk, needs to be downloaded) + + FileReference fileReference = new FileReference("fileReference"); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); + assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + + // Verify download status + assertDownloadStatus(fileDownloader, fileReference, 0.0); + + // Receives fileReference, should return and make it available to caller + String filename = "abc.jar"; + receiveFile(fileReference, filename, "some other content"); + Optional<File> downloadedFile = fileDownloader.getFile(fileReference); + + assertTrue(downloadedFile.isPresent()); + File downloadedFileFullPath = new File(fileReferenceFullPath, filename); + assertEquals(downloadedFileFullPath.getAbsolutePath(), downloadedFile.get().getAbsolutePath()); + assertEquals("some other content", IOUtils.readFile(downloadedFile.get())); + + // Verify download status when downloaded + assertDownloadStatus(fileDownloader, fileReference, 100.0); + } + } + + @Test + public void setFilesToDownload() throws IOException { + Duration timeout = Duration.ofMillis(200); + File downloadDir = Files.createTempDirectory("filedistribution").toFile(); + MockConnection connectionPool = new MockConnection(); + connectionPool.setResponseHandler(new MockConnection.WaitResponseHandler(timeout.plus(Duration.ofMillis(1000)))); + FileDownloader fileDownloader = new FileDownloader(connectionPool, downloadDir, timeout); + FileReference foo = new FileReference("foo"); + FileReference bar = new FileReference("bar"); + List<FileReference> fileReferences = Arrays.asList(foo, bar); + fileDownloader.queueForDownload(fileReferences); + + // Verify download status + assertDownloadStatus(fileDownloader, foo, 0.0); + assertDownloadStatus(fileDownloader, bar, 0.0); + } + + @Test + public void receiveFile() throws IOException { + FileReference foo = new FileReference("foo"); + String filename = "foo.jar"; + receiveFile(foo, filename, "content"); + File downloadedFile = new File(fileReferenceFullPath(downloadDir, foo), filename); + assertEquals("content", IOUtils.readFile(downloadedFile)); + } + + private void writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException { + File file = new File(new File(dir, fileReferenceString), fileName); + IOUtils.writeFile(file, "content", false); + } + + private File fileReferenceFullPath(File dir, FileReference fileReference) { + return new File(dir, fileReference.value()); + } + + private void assertDownloadStatus(FileDownloader fileDownloader, FileReference fileReference, double expectedDownloadStatus) { + double downloadStatus = fileDownloader.downloadStatus(fileReference); + assertEquals(expectedDownloadStatus, downloadStatus, 0.0001); + } + + private void receiveFile(FileReference fileReference, String filename, String content) { + byte[] contentBytes = Utf8.toBytes(content); + long xxHashFromContent = hasher.hash(ByteBuffer.wrap(contentBytes), 0); + fileDownloader.receiveFile(fileReference, filename, contentBytes, xxHashFromContent); + } + + private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { + + private ResponseHandler responseHandler; + + MockConnection() { + this(new FileReferenceFoundResponseHandler()); + } + + MockConnection(ResponseHandler responseHandler) { + this.responseHandler = responseHandler; + } + + @Override + public void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter) { + responseHandler.request(request); + } + + @Override + public void invokeSync(Request request, double jrtTimeout) { + responseHandler.request(request); + } + + @Override + public void setError(int errorCode) { + } + + @Override + public void setSuccess() { + } + + @Override + public String getAddress() { + return null; + } + + @Override + public void close() { + } + + @Override + public void setError(Connection connection, int errorCode) { + connection.setError(errorCode); + } + + @Override + public Connection getCurrent() { + return this; + } + + @Override + public Connection setNewCurrentConnection() { + return this; + } + + @Override + public int getSize() { + return 1; + } + + @Override + public Supervisor getSupervisor() { + return new Supervisor(new Transport()); + } + + void setResponseHandler(ResponseHandler responseHandler) { + this.responseHandler = responseHandler; + } + + public interface ResponseHandler { + void request(Request request); + } + + static class FileReferenceFoundResponseHandler implements MockConnection.ResponseHandler { + + @Override + public void request(Request request) { + if (request.methodName().equals("filedistribution.serveFile")) { + request.returnValues().add(new Int32Value(0)); + request.returnValues().add(new StringValue("OK")); + } + } + } + + static class UnknownFileReferenceResponseHandler implements MockConnection.ResponseHandler { + + @Override + public void request(Request request) { + if (request.methodName().equals("filedistribution.serveFile")) { + request.returnValues().add(new Int32Value(1)); + request.returnValues().add(new StringValue("Internal error")); + } + } + } + + static class WaitResponseHandler implements MockConnection.ResponseHandler { + + private final Duration waitUntilAnswering; + + WaitResponseHandler(Duration waitUntilAnswering) { + super(); + this.waitUntilAnswering = waitUntilAnswering; + } + + @Override + public void request(Request request) { + try { Thread.sleep(waitUntilAnswering.toMillis());} catch (InterruptedException e) { /* do nothing */ } + + if (request.methodName().equals("filedistribution.serveFile")) { + request.returnValues().add(new Int32Value(0)); + request.returnValues().add(new StringValue("OK")); + } + + } + } + } + +} |