From ba160b5f48616194aff36b5256bb16d440784769 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 22 Nov 2017 11:04:22 +0100 Subject: Revert "Revert "Revert "Add support for downloading from another config server""" --- .../filedistribution/FileReferenceDownloader.java | 30 ++++------- .../vespa/filedistribution/FileDownloaderTest.java | 58 +--------------------- 2 files changed, 10 insertions(+), 78 deletions(-) (limited to 'filedistribution') diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 08595662f36..fbadddd624a 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; -import com.yahoo.jrt.ErrorCode; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; import com.yahoo.log.LogLevel; @@ -36,6 +35,7 @@ import java.util.stream.Collectors; * * @author hmusum */ +// TODO: Add retries when a config server does not have a file reference // TODO: Handle shutdown of executors class FileReferenceDownloader { @@ -66,20 +66,10 @@ class FileReferenceDownloader { throws ExecutionException, InterruptedException, TimeoutException { downloads.put(fileReference, fileReferenceDownload); setDownloadStatus(fileReference.value(), 0.0); - - int numAttempts = 0; - boolean downloadStarted = false; - do { - if (startDownloadRpc(fileReference)) - downloadStarted = true; - else - Thread.sleep(100); - } while (!downloadStarted && ++numAttempts <= 10); // TODO: How long/many times to retry? - - if (downloadStarted) { + if (startDownloadRpc(fileReference)) return fileReferenceDownload.future().get(timeout.toMillis(), TimeUnit.MILLISECONDS); - } else { - fileReferenceDownload.future().setException(new RuntimeException("Failed getting file reference '" + fileReference.value() + "'")); + else { + fileReferenceDownload.future().setException(new RuntimeException("Failed getting file")); downloads.remove(fileReference); return Optional.empty(); } @@ -128,17 +118,15 @@ class FileReferenceDownloader { execute(request, connection); if (validateResponse(request)) { log.log(LogLevel.DEBUG, "Request callback, OK. Req: " + request + "\nSpec: " + connection); - if (request.returnValues().get(0).asInt32() == 0) { + if (request.returnValues().get(0).asInt32() == 0) log.log(LogLevel.INFO, "Found file reference '" + fileReference.value() + "' available at " + connection.getAddress()); - return true; - } else { + else log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found for " + connection.getAddress()); - return false; - } + return true; } else { log.log(LogLevel.WARNING, "Request failed. Req: " + request + "\nSpec: " + connection.getAddress()); - if (request.isError() && request.errorCode() == ErrorCode.CONNECTION) - connection.setError(request.errorCode()); + connection.setError(request.errorCode()); + // TODO: Retry with another config server return false; } } diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index 278c46dab8b..738b0888956 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -27,7 +27,6 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; -import static com.yahoo.jrt.ErrorCode.CONNECTION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -46,7 +45,7 @@ public class FileDownloaderTest { try { downloadDir = Files.createTempDirectory("filedistribution").toFile(); connection = new MockConnection(); - fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(2000)); + fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000)); } catch (IOException e) { e.printStackTrace(); fail(e.getMessage()); @@ -115,38 +114,6 @@ public class FileDownloaderTest { } } - @Test - public void getFileWhenConnectionError() throws IOException { - fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000)); - File downloadDir = fileDownloader.downloadDirectory(); - - int timesToFail = 2; - MockConnection.ConnectionErrorResponseHandler responseHandler = new MockConnection.ConnectionErrorResponseHandler(timesToFail); - connection.setResponseHandler(responseHandler); - - FileReference fileReference = new FileReference("fileReference"); - File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); - assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); - - // Verify download status - assertDownloadStatus(fileDownloader, fileReference, 0.0); - - // Receives fileReference, should return and make it available to caller - String filename = "abc.jar"; - receiveFile(fileReference, filename, "some other content"); - Optional downloadedFile = fileDownloader.getFile(fileReference); - - assertTrue(downloadedFile.isPresent()); - File downloadedFileFullPath = new File(fileReferenceFullPath, filename); - assertEquals(downloadedFileFullPath.getAbsolutePath(), downloadedFile.get().getAbsolutePath()); - assertEquals("some other content", IOUtils.readFile(downloadedFile.get())); - - // Verify download status when downloaded - assertDownloadStatus(fileDownloader, fileReference, 100.0); - - assertEquals(timesToFail, responseHandler.failedTimes); - } - @Test public void setFilesToDownload() throws IOException { Duration timeout = Duration.ofMillis(200); @@ -304,30 +271,7 @@ public class FileDownloaderTest { request.returnValues().add(new Int32Value(0)); request.returnValues().add(new StringValue("OK")); } - } - } - - static class ConnectionErrorResponseHandler implements MockConnection.ResponseHandler { - private final int timesToFail; - private int failedTimes = 0; - - ConnectionErrorResponseHandler(int timesToFail) { - super(); - this.timesToFail = timesToFail; - } - - @Override - public void request(Request request) { - if (request.methodName().equals("filedistribution.serveFile")) { - if (failedTimes < timesToFail) { - request.setError(CONNECTION, "Connection error"); - failedTimes++; - } else { - request.returnValues().add(new Int32Value(0)); - request.returnValues().add(new StringValue("OK")); - } - } } } } -- cgit v1.2.3