More testing of download

4 files changed, 47 insertions, 16 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
index 9d35b4f0e32..a89d5a232ad 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
@@ -68,16 +68,17 @@ public class FileServer {
     @SuppressWarnings("WeakerAccess") // Created by dependency injection
     @Inject
     public FileServer(ConfigserverConfig configserverConfig) {
-        this(createConnectionPool(configserverConfig), new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())));
+        this(new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())),
+             new FileDownloader(createConnectionPool(configserverConfig)));
     }
 
     // For testing only
     public FileServer(File rootDir) {
-        this(emptyConnectionPool(), rootDir);
+        this(rootDir, new FileDownloader(emptyConnectionPool()));
     }
 
-    private FileServer(ConnectionPool connectionPool, File rootDir) {
-        this.downloader = new FileDownloader(connectionPool);
+    public FileServer(File rootDir, FileDownloader fileDownloader) {
+        this.downloader = fileDownloader;
         this.root = new FileDirectory(rootDir);
         this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()),
                                                          new DaemonThreadFactory("file server push"));
@@ -176,7 +177,7 @@ public class FileServer {
 
     // downloadFromOtherSourceIfNotFound is true when the request comes from another config server.
     // This is to avoid config servers asking each other for a file that does not exist
-    private boolean download(FileReferenceDownload fileReferenceDownload) {
+    boolean download(FileReferenceDownload fileReferenceDownload) {
         if (fileReferenceDownload.downloadFromOtherSourceIfNotFound()) {
             log.log(Level.FINE, "File not found, downloading from another source");
             return downloader.getFile(fileReferenceDownload).isPresent();
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
index 2facca99f60..9a96c627db1 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
@@ -1,10 +1,13 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.config.FileReference;
 import com.yahoo.io.IOUtils;
 import com.yahoo.net.HostName;
+import com.yahoo.vespa.filedistribution.FileDownloader;
 import com.yahoo.vespa.filedistribution.FileReferenceData;
+import com.yahoo.vespa.filedistribution.FileReferenceDownload;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -12,14 +15,16 @@ import org.junit.rules.TemporaryFolder;
 
 import java.io.File;
 import java.io.IOException;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 
+import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.emptyConnectionPool;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 public class FileServerTest {
 
@@ -31,23 +36,32 @@ public class FileServerTest {
     @Before
     public void setup() throws IOException {
         File rootDir = new File(temporaryFolder.newFolder("fileserver-root").getAbsolutePath());
-        fileServer = new FileServer(rootDir);
+        fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir));
     }
 
     @Test
-    public void requireThatExistingFileCanBeFound() throws IOException {
-        File dir = getFileServerRootDir();
-        IOUtils.createDirectory(dir + "/123");
-        IOUtils.writeFile(dir + "/123/f1", "test", false);
-        assertTrue(fileServer.hasFile("123"));
+    public void requireThatExistingFileIsFound() throws IOException {
+        String dir = "123";
+        writeFile(dir);
+        assertTrue(fileServer.hasFile(dir));
     }
 
     @Test
-    public void requireThatNonExistingFileCanNotBeFound() {
+    public void requireThatNonExistingFileIsNotFound() {
         assertFalse(fileServer.hasFile("12x"));
     }
 
     @Test
+    public void requireThatNonExistingFileWillBeDownloaded() throws IOException {
+        String dir = "123";
+        assertFalse(fileServer.hasFile(dir));
+        FileReferenceDownload foo = new FileReferenceDownload(new FileReference(dir), true, "foo");
+        assertFalse(fileServer.download(foo));
+        writeFile(dir);
+        assertTrue(fileServer.download(foo));
+    }
+
+    @Test
     public void requireThatFileReferenceWithDirectoryCanBeFound() throws IOException {
         File dir = getFileServerRootDir();
         IOUtils.writeFile(dir + "/124/subdir/f1", "test", false);
@@ -93,6 +107,12 @@ public class FileServerTest {
         assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize());
     }
 
+    private void writeFile(String dir) throws IOException {
+        File rootDir = getFileServerRootDir();
+        IOUtils.createDirectory(rootDir + "/" + dir);
+        IOUtils.writeFile(rootDir + "/" + dir + "/f1", "test", false);
+    }
+
     private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) throws IOException {
         File fileReferencesDir = temporaryFolder.newFolder();
         configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath());
@@ -114,4 +134,12 @@ public class FileServerTest {
         return fileServer.getRootDir().getRoot();
     }
 
+    private static class MockFileDownloader extends FileDownloader {
+
+        public MockFileDownloader(File downloadDirectory) {
+            super(emptyConnectionPool(), downloadDirectory, downloadDirectory, Duration.ofMillis(100), Duration.ofMillis(100));
+        }
+
+    }
+
 }
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
index 7470ac0952d..beed82ad931 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
@@ -42,7 +42,7 @@ public class FileDownloader implements AutoCloseable {
         this(connectionPool, downloadDirectory , downloadDirectory , Duration.ofMinutes(15), Duration.ofSeconds(10));
     }
 
-    FileDownloader(ConnectionPool connectionPool, File downloadDirectory, File tmpDirectory, Duration timeout, Duration sleepBetweenRetries) {
+    public FileDownloader(ConnectionPool connectionPool, File downloadDirectory, File tmpDirectory, Duration timeout, Duration sleepBetweenRetries) {
         this.downloadDirectory = downloadDirectory;
         this.timeout = timeout;
         this.fileReferenceDownloader = new FileReferenceDownloader(downloadDirectory, tmpDirectory, connectionPool, timeout, sleepBetweenRetries);
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java
index 31e6eb5daab..d927063ba60 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java
@@ -28,7 +28,7 @@ public class FileReferenceDownload {
         this.client = client;
     }
 
-    FileReference fileReference() {
+    public FileReference fileReference() {
         return fileReference;
     }
 
@@ -40,6 +40,8 @@ public class FileReferenceDownload {
         return downloadFromOtherSourceIfNotFound;
     }
 
+    public String client() { return client;  }
+
     @Override
     public String toString() {
         return fileReference + ", client: " + client;