From 982da2851d9390a1a06c7fa3686f2500671c4d61 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 15 Oct 2020 11:21:10 +0200 Subject: More testing of download --- .../config/server/filedistribution/FileServer.java | 11 +++--- .../server/filedistribution/FileServerTest.java | 46 +++++++++++++++++----- 2 files changed, 43 insertions(+), 14 deletions(-) (limited to 'configserver/src') diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 9d35b4f0e32..a89d5a232ad 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -68,16 +68,17 @@ public class FileServer { @SuppressWarnings("WeakerAccess") // Created by dependency injection @Inject public FileServer(ConfigserverConfig configserverConfig) { - this(createConnectionPool(configserverConfig), new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()))); + this(new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())), + new FileDownloader(createConnectionPool(configserverConfig))); } // For testing only public FileServer(File rootDir) { - this(emptyConnectionPool(), rootDir); + this(rootDir, new FileDownloader(emptyConnectionPool())); } - private FileServer(ConnectionPool connectionPool, File rootDir) { - this.downloader = new FileDownloader(connectionPool); + public FileServer(File rootDir, FileDownloader fileDownloader) { + this.downloader = fileDownloader; this.root = new FileDirectory(rootDir); this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("file server push")); @@ -176,7 +177,7 @@ public class FileServer { // downloadFromOtherSourceIfNotFound is true when the request comes from another config server. // This is to avoid config servers asking each other for a file that does not exist - private boolean download(FileReferenceDownload fileReferenceDownload) { + boolean download(FileReferenceDownload fileReferenceDownload) { if (fileReferenceDownload.downloadFromOtherSourceIfNotFound()) { log.log(Level.FINE, "File not found, downloading from another source"); return downloader.getFile(fileReferenceDownload).isPresent(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index 2facca99f60..9a96c627db1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -1,10 +1,13 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.FileReference; import com.yahoo.io.IOUtils; import com.yahoo.net.HostName; +import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceData; +import com.yahoo.vespa.filedistribution.FileReferenceDownload; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -12,14 +15,16 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.emptyConnectionPool; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class FileServerTest { @@ -31,22 +36,31 @@ public class FileServerTest { @Before public void setup() throws IOException { File rootDir = new File(temporaryFolder.newFolder("fileserver-root").getAbsolutePath()); - fileServer = new FileServer(rootDir); + fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir)); } @Test - public void requireThatExistingFileCanBeFound() throws IOException { - File dir = getFileServerRootDir(); - IOUtils.createDirectory(dir + "/123"); - IOUtils.writeFile(dir + "/123/f1", "test", false); - assertTrue(fileServer.hasFile("123")); + public void requireThatExistingFileIsFound() throws IOException { + String dir = "123"; + writeFile(dir); + assertTrue(fileServer.hasFile(dir)); } @Test - public void requireThatNonExistingFileCanNotBeFound() { + public void requireThatNonExistingFileIsNotFound() { assertFalse(fileServer.hasFile("12x")); } + @Test + public void requireThatNonExistingFileWillBeDownloaded() throws IOException { + String dir = "123"; + assertFalse(fileServer.hasFile(dir)); + FileReferenceDownload foo = new FileReferenceDownload(new FileReference(dir), true, "foo"); + assertFalse(fileServer.download(foo)); + writeFile(dir); + assertTrue(fileServer.download(foo)); + } + @Test public void requireThatFileReferenceWithDirectoryCanBeFound() throws IOException { File dir = getFileServerRootDir(); @@ -93,6 +107,12 @@ public class FileServerTest { assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); } + private void writeFile(String dir) throws IOException { + File rootDir = getFileServerRootDir(); + IOUtils.createDirectory(rootDir + "/" + dir); + IOUtils.writeFile(rootDir + "/" + dir + "/f1", "test", false); + } + private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) throws IOException { File fileReferencesDir = temporaryFolder.newFolder(); configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath()); @@ -114,4 +134,12 @@ public class FileServerTest { return fileServer.getRootDir().getRoot(); } + private static class MockFileDownloader extends FileDownloader { + + public MockFileDownloader(File downloadDirectory) { + super(emptyConnectionPool(), downloadDirectory, downloadDirectory, Duration.ofMillis(100), Duration.ofMillis(100)); + } + + } + } -- cgit v1.2.3