summaryrefslogtreecommitdiffstats
path: root/filedistribution
diff options
context:
space:
mode:
authorHarald Musum <musum@yahoo-inc.com>2017-11-21 16:27:53 +0100
committerGitHub <noreply@github.com>2017-11-21 16:27:53 +0100
commitde9ca41a485dde31d498c5e8522e9fcbe21189ea (patch)
tree32485e2b241156dfe7e581e236ed6ad6c0cf6c00 /filedistribution
parentc0250c086cebd1c8dba972339f31715697d58f84 (diff)
Revert "Add support for downloading from another config server"
Diffstat (limited to 'filedistribution')
-rw-r--r--filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java30
-rw-r--r--filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java58
2 files changed, 10 insertions, 78 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
index 08595662f36..fbadddd624a 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
@@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.ListenableFuture;
import com.yahoo.concurrent.DaemonThreadFactory;
import com.yahoo.config.FileReference;
-import com.yahoo.jrt.ErrorCode;
import com.yahoo.jrt.Request;
import com.yahoo.jrt.StringValue;
import com.yahoo.log.LogLevel;
@@ -36,6 +35,7 @@ import java.util.stream.Collectors;
*
* @author hmusum
*/
+// TODO: Add retries when a config server does not have a file reference
// TODO: Handle shutdown of executors
class FileReferenceDownloader {
@@ -66,20 +66,10 @@ class FileReferenceDownloader {
throws ExecutionException, InterruptedException, TimeoutException {
downloads.put(fileReference, fileReferenceDownload);
setDownloadStatus(fileReference.value(), 0.0);
-
- int numAttempts = 0;
- boolean downloadStarted = false;
- do {
- if (startDownloadRpc(fileReference))
- downloadStarted = true;
- else
- Thread.sleep(100);
- } while (!downloadStarted && ++numAttempts <= 10); // TODO: How long/many times to retry?
-
- if (downloadStarted) {
+ if (startDownloadRpc(fileReference))
return fileReferenceDownload.future().get(timeout.toMillis(), TimeUnit.MILLISECONDS);
- } else {
- fileReferenceDownload.future().setException(new RuntimeException("Failed getting file reference '" + fileReference.value() + "'"));
+ else {
+ fileReferenceDownload.future().setException(new RuntimeException("Failed getting file"));
downloads.remove(fileReference);
return Optional.empty();
}
@@ -128,17 +118,15 @@ class FileReferenceDownloader {
execute(request, connection);
if (validateResponse(request)) {
log.log(LogLevel.DEBUG, "Request callback, OK. Req: " + request + "\nSpec: " + connection);
- if (request.returnValues().get(0).asInt32() == 0) {
+ if (request.returnValues().get(0).asInt32() == 0)
log.log(LogLevel.INFO, "Found file reference '" + fileReference.value() + "' available at " + connection.getAddress());
- return true;
- } else {
+ else
log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found for " + connection.getAddress());
- return false;
- }
+ return true;
} else {
log.log(LogLevel.WARNING, "Request failed. Req: " + request + "\nSpec: " + connection.getAddress());
- if (request.isError() && request.errorCode() == ErrorCode.CONNECTION)
- connection.setError(request.errorCode());
+ connection.setError(request.errorCode());
+ // TODO: Retry with another config server
return false;
}
}
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
index 278c46dab8b..738b0888956 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
@@ -27,7 +27,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.Optional;
-import static com.yahoo.jrt.ErrorCode.CONNECTION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -46,7 +45,7 @@ public class FileDownloaderTest {
try {
downloadDir = Files.createTempDirectory("filedistribution").toFile();
connection = new MockConnection();
- fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(2000));
+ fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000));
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
@@ -116,38 +115,6 @@ public class FileDownloaderTest {
}
@Test
- public void getFileWhenConnectionError() throws IOException {
- fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000));
- File downloadDir = fileDownloader.downloadDirectory();
-
- int timesToFail = 2;
- MockConnection.ConnectionErrorResponseHandler responseHandler = new MockConnection.ConnectionErrorResponseHandler(timesToFail);
- connection.setResponseHandler(responseHandler);
-
- FileReference fileReference = new FileReference("fileReference");
- File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference);
- assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent());
-
- // Verify download status
- assertDownloadStatus(fileDownloader, fileReference, 0.0);
-
- // Receives fileReference, should return and make it available to caller
- String filename = "abc.jar";
- receiveFile(fileReference, filename, "some other content");
- Optional<File> downloadedFile = fileDownloader.getFile(fileReference);
-
- assertTrue(downloadedFile.isPresent());
- File downloadedFileFullPath = new File(fileReferenceFullPath, filename);
- assertEquals(downloadedFileFullPath.getAbsolutePath(), downloadedFile.get().getAbsolutePath());
- assertEquals("some other content", IOUtils.readFile(downloadedFile.get()));
-
- // Verify download status when downloaded
- assertDownloadStatus(fileDownloader, fileReference, 100.0);
-
- assertEquals(timesToFail, responseHandler.failedTimes);
- }
-
- @Test
public void setFilesToDownload() throws IOException {
Duration timeout = Duration.ofMillis(200);
File downloadDir = Files.createTempDirectory("filedistribution").toFile();
@@ -304,30 +271,7 @@ public class FileDownloaderTest {
request.returnValues().add(new Int32Value(0));
request.returnValues().add(new StringValue("OK"));
}
- }
- }
-
- static class ConnectionErrorResponseHandler implements MockConnection.ResponseHandler {
- private final int timesToFail;
- private int failedTimes = 0;
-
- ConnectionErrorResponseHandler(int timesToFail) {
- super();
- this.timesToFail = timesToFail;
- }
-
- @Override
- public void request(Request request) {
- if (request.methodName().equals("filedistribution.serveFile")) {
- if (failedTimes < timesToFail) {
- request.setError(CONNECTION, "Connection error");
- failedTimes++;
- } else {
- request.returnValues().add(new Int32Value(0));
- request.returnValues().add(new StringValue("OK"));
- }
- }
}
}
}