Revert "Revert "Add support for downloading from another config server""

2 files changed, 78 insertions, 10 deletions

diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
index fbadddd624a..08595662f36 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
@@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.FileReference;
+import com.yahoo.jrt.ErrorCode;
 import com.yahoo.jrt.Request;
 import com.yahoo.jrt.StringValue;
 import com.yahoo.log.LogLevel;
@@ -35,7 +36,6 @@ import java.util.stream.Collectors;
  *
  * @author hmusum
  */
-// TODO: Add retries when a config server does not have a file reference
 // TODO: Handle shutdown of executors
 class FileReferenceDownloader {
 
@@ -66,10 +66,20 @@ class FileReferenceDownloader {
             throws ExecutionException, InterruptedException, TimeoutException {
         downloads.put(fileReference, fileReferenceDownload);
         setDownloadStatus(fileReference.value(), 0.0);
-        if (startDownloadRpc(fileReference))
+
+        int numAttempts = 0;
+        boolean downloadStarted = false;
+        do {
+            if (startDownloadRpc(fileReference))
+                downloadStarted = true;
+            else
+                Thread.sleep(100);
+        } while (!downloadStarted && ++numAttempts <= 10);  // TODO: How long/many times to retry?
+
+        if (downloadStarted) {
             return fileReferenceDownload.future().get(timeout.toMillis(), TimeUnit.MILLISECONDS);
-        else {
-            fileReferenceDownload.future().setException(new RuntimeException("Failed getting file"));
+        } else {
+            fileReferenceDownload.future().setException(new RuntimeException("Failed getting file reference '" + fileReference.value() + "'"));
             downloads.remove(fileReference);
             return Optional.empty();
         }
@@ -118,15 +128,17 @@ class FileReferenceDownloader {
         execute(request, connection);
         if (validateResponse(request)) {
             log.log(LogLevel.DEBUG, "Request callback, OK. Req: " + request + "\nSpec: " + connection);
-            if (request.returnValues().get(0).asInt32() == 0)
+            if (request.returnValues().get(0).asInt32() == 0) {
                 log.log(LogLevel.INFO, "Found file reference '" + fileReference.value() + "' available at " + connection.getAddress());
-            else
+                return true;
+            } else {
                 log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found for " + connection.getAddress());
-            return true;
+                return false;
+            }
         } else {
             log.log(LogLevel.WARNING, "Request failed. Req: " + request + "\nSpec: " + connection.getAddress());
-            connection.setError(request.errorCode());
-            // TODO: Retry with another config server
+            if (request.isError() && request.errorCode() == ErrorCode.CONNECTION)
+                connection.setError(request.errorCode());
             return false;
         }
     }
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
index 738b0888956..278c46dab8b 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
 
+import static com.yahoo.jrt.ErrorCode.CONNECTION;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -45,7 +46,7 @@ public class FileDownloaderTest {
         try {
             downloadDir = Files.createTempDirectory("filedistribution").toFile();
             connection = new MockConnection();
-            fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000));
+            fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(2000));
         } catch (IOException e) {
             e.printStackTrace();
             fail(e.getMessage());
@@ -115,6 +116,38 @@ public class FileDownloaderTest {
     }
 
     @Test
+    public void getFileWhenConnectionError() throws IOException {
+        fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000));
+        File downloadDir = fileDownloader.downloadDirectory();
+
+        int timesToFail = 2;
+        MockConnection.ConnectionErrorResponseHandler responseHandler = new MockConnection.ConnectionErrorResponseHandler(timesToFail);
+        connection.setResponseHandler(responseHandler);
+
+        FileReference fileReference = new FileReference("fileReference");
+        File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference);
+        assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent());
+
+        // Verify download status
+        assertDownloadStatus(fileDownloader, fileReference, 0.0);
+
+        // Receives fileReference, should return and make it available to caller
+        String filename = "abc.jar";
+        receiveFile(fileReference, filename, "some other content");
+        Optional<File> downloadedFile = fileDownloader.getFile(fileReference);
+
+        assertTrue(downloadedFile.isPresent());
+        File downloadedFileFullPath = new File(fileReferenceFullPath, filename);
+        assertEquals(downloadedFileFullPath.getAbsolutePath(), downloadedFile.get().getAbsolutePath());
+        assertEquals("some other content", IOUtils.readFile(downloadedFile.get()));
+
+        // Verify download status when downloaded
+        assertDownloadStatus(fileDownloader, fileReference, 100.0);
+
+        assertEquals(timesToFail, responseHandler.failedTimes);
+    }
+
+    @Test
     public void setFilesToDownload() throws IOException {
         Duration timeout = Duration.ofMillis(200);
         File downloadDir = Files.createTempDirectory("filedistribution").toFile();
@@ -271,7 +304,30 @@ public class FileDownloaderTest {
                     request.returnValues().add(new Int32Value(0));
                     request.returnValues().add(new StringValue("OK"));
                 }
+            }
+        }
+
+        static class ConnectionErrorResponseHandler implements MockConnection.ResponseHandler {
 
+            private final int timesToFail;
+            private int failedTimes = 0;
+
+            ConnectionErrorResponseHandler(int timesToFail) {
+                super();
+                this.timesToFail = timesToFail;
+            }
+
+            @Override
+            public void request(Request request) {
+                if (request.methodName().equals("filedistribution.serveFile")) {
+                    if (failedTimes < timesToFail) {
+                        request.setError(CONNECTION, "Connection error");
+                        failedTimes++;
+                    } else {
+                        request.returnValues().add(new Int32Value(0));
+                        request.returnValues().add(new StringValue("OK"));
+                    }
+                }
             }
         }
     }