Implement downloading of file references, not functional yet

4 files changed, 192 insertions, 40 deletions

diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
index c1e9826e29f..f9b334a6f87 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
@@ -38,8 +38,7 @@ public class ConfigProxyRpcServerTest {
 
     @Test
     public void basic() {
-        ConfigSourceSet configSources = new ConfigSourceSet();
-        ProxyServer proxy = ProxyServer.createTestServer(configSources);
+        ProxyServer proxy = ProxyServer.createTestServer(new MockConfigSource(new MockClientUpdater()));
         Spec spec = new Spec("localhost", 12345);
         ConfigProxyRpcServer server = new ConfigProxyRpcServer(proxy, spec);
         assertThat(server.getSpec(), is(spec));
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java
index dc9a3408510..2b26996fbdc 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java
@@ -1,12 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.proxy;
 
-import com.yahoo.config.subscription.ConfigSource;
 import com.yahoo.config.subscription.ConfigSourceSet;
 import com.yahoo.vespa.config.ConfigKey;
 import com.yahoo.vespa.config.RawConfig;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.Set;
 
 /**
  * A simple class to be able to test config proxy without having an RPC config
@@ -37,4 +39,9 @@ class MockConfigSource extends ConfigSourceSet {
         backing.clear();
     }
 
+    @Override
+    public Set<String> getSources() {
+        return Collections.singleton("tcp/localhost:19070,tcp/localhost:19071,tcp/localhost:19072");
+    }
+
 }
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
index f82d9e90184..3cd0f1043cc 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
@@ -95,7 +95,6 @@ public class ProxyServerTest {
      */
     @Test
     public void testModeSwitch() {
-        ConfigSourceSet source = new ConfigSourceSet(); // Need to use a ConfigSourceSet to test modes
         ProxyServer proxy = ProxyServer.createTestServer(source);
         assertTrue(proxy.getMode().isDefault());
 
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java
index ea880e451b6..64ae1a07aea 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java
@@ -1,79 +1,226 @@
 package com.yahoo.vespa.config.proxy.filedistribution;
 
 import com.yahoo.config.FileReference;
-import com.yahoo.config.subscription.ConfigSourceSet;
 import com.yahoo.io.IOUtils;
+import com.yahoo.jrt.Int32Value;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.RequestWaiter;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.config.Connection;
+import com.yahoo.vespa.config.ConnectionPool;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.time.Duration;
-import java.util.*;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 public class FileDownloaderTest {
-    private static final ConfigSourceSet configSourceSet = new ConfigSourceSet();
+
+    private MockConnection connection;
+    private FileDownloader fileDownloader;
+
+    @Before
+    public void setup() {
+        try {
+            File downloadDir = Files.createTempDirectory("filedistribution").toFile();
+            connection = new MockConnection();
+            fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000));
+        } catch (IOException e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
 
     @Test
-    public void download() throws IOException {
-        File downloadDir = Files.createTempDirectory("filedistribution").toFile();
-        FileDownloader fileDownloader = new FileDownloader(configSourceSet, downloadDir.getAbsolutePath(), Duration.ofMillis(200));
+    public void getFile() throws IOException {
+        File downloadDir = fileDownloader.downloadDirectory();
 
-        // Write a file to download directory to simulate download going OK
-        String fileReferenceString = "somehash";
-        String fileName = "foo.jar";
-        File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReferenceString);
-        FileReference fileReference = writeFileReference(downloadDir, fileReferenceString, fileName);
+        {
+            // fileReference already exists on disk, does not have to be downloaded
 
-        // Check that we get correct path and content when asking for file reference
-        Optional<File> pathToFile = fileDownloader.getFile(fileReference);
-        assertTrue(pathToFile.isPresent());
-        String downloadedFile = new File(fileReferenceFullPath, fileName).getAbsolutePath();
-        assertEquals(new File(fileReferenceFullPath, fileName).getAbsolutePath(), downloadedFile);
-        assertEquals("content", IOUtils.readFile(pathToFile.get()));
+            String fileReferenceString = "foo";
+            String filename = "foo.jar";
+            File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReferenceString);
+            FileReference fileReference = new FileReference(fileReferenceString);
+            writeFileReference(downloadDir, fileReferenceString, filename);
 
-        // Verify download status
-        Map<FileReference, Double> downloadStatus = fileDownloader.downloadStatus();
-        assertEquals(1, downloadStatus.size());
-        assertDownloadStatus(Collections.singletonList(fileReference), downloadStatus.entrySet().iterator().next(), 100.0);
+            // Check that we get correct path and content when asking for file reference
+            Optional<File> pathToFile = fileDownloader.getFile(fileReference);
+            assertTrue(pathToFile.isPresent());
+            String downloadedFile = new File(fileReferenceFullPath, filename).getAbsolutePath();
+            assertEquals(new File(fileReferenceFullPath, filename).getAbsolutePath(), downloadedFile);
+            assertEquals("content", IOUtils.readFile(pathToFile.get()));
+
+            // Verify download status when downloaded
+            assertDownloadStatus(fileDownloader, fileReference, 100.0);
+        }
+
+        {
+            // fileReference does not exist on disk, needs to be downloaded, but fails when asking upstream for file)
+
+            connection.setResponseHandler(new MockConnection.UnknownFileReferenceResponseHandler());
+
+            FileReference fileReference = new FileReference("bar");
+            File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference.value());
+            assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent());
+
+            // Verify download status when unable to download
+            assertDownloadStatus(fileDownloader, fileReference, 0.0);
+        }
+
+        {
+            // fileReference does not exist on disk, needs to be downloaded)
 
-        // Non-existing file
-        assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(new FileReference("doesnotexist")).isPresent());
+            FileReference fileReference = new FileReference("fileReference");
+            File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference.value());
+            assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent());
+
+            // Verify download status
+            assertDownloadStatus(fileDownloader, fileReference, 0.0);
+
+            // Receives fileReference, should return and make it available to caller
+            String filename = "abc.jar";
+            fileDownloader.receiveFile(fileReference, filename, Utf8.toBytes("some other content"));
+            Optional<File> downloadedFile = fileDownloader.getFile(fileReference);
+
+            assertTrue(downloadedFile.isPresent());
+            File downloadedFileFullPath = new File(fileReferenceFullPath, filename);
+            assertEquals(downloadedFileFullPath.getAbsolutePath(), downloadedFile.get().getAbsolutePath());
+            assertEquals("some other content", IOUtils.readFile(downloadedFile.get()));
+
+            // Verify download status when downloaded
+            assertDownloadStatus(fileDownloader, fileReference, 100.0);
+        }
     }
 
     @Test
     public void setFilesToDownload() throws IOException {
         File downloadDir = Files.createTempDirectory("filedistribution").toFile();
-        FileDownloader fileDownloader = new FileDownloader(configSourceSet, downloadDir.getAbsolutePath(), Duration.ofMillis(200));
-        List<FileReference> fileReferences = Arrays.asList(new FileReference("foo"), new FileReference("bar"));
-        fileDownloader.queueForDownload(fileReferences);
+        MockConnection configSource = new MockConnection();
+        FileDownloader fileDownloader = new FileDownloader(configSource, downloadDir, Duration.ofMillis(200));
+        FileReference foo = new FileReference("foo");
+        FileReference bar = new FileReference("bar");
+        List<FileReference> fileReferences = Arrays.asList(foo, bar);
+        fileDownloader.download(fileReferences);
 
         assertEquals(fileReferences, fileDownloader.queuedForDownload().asList());
 
         // Verify download status
-        Map<FileReference, Double> downloadStatus = fileDownloader.downloadStatus();
-        assertEquals(2, downloadStatus.size());
-
-        assertDownloadStatus(fileReferences, downloadStatus.entrySet().iterator().next(), 0.0);
-        assertDownloadStatus(fileReferences, downloadStatus.entrySet().iterator().next(), 0.0);
+        assertDownloadStatus(fileDownloader, foo, 0.0);
+        assertDownloadStatus(fileDownloader, bar, 0.0);
     }
 
-    private FileReference writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException {
+    private void writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException {
         File file = new File(new File(dir, fileReferenceString), fileName);
         IOUtils.writeFile(file, "content", false);
-        return new FileReference(fileReferenceString);
     }
 
     private File fileReferenceFullPath(File dir, String fileReferenceString) {
         return new File(dir, fileReferenceString);
     }
 
-    private void assertDownloadStatus(List<FileReference> fileReferences, Map.Entry<FileReference, Double> entry, double expectedDownloadStatus) {
-        assertTrue(fileReferences.contains(new FileReference(entry.getKey().value())));
-        assertEquals(expectedDownloadStatus, entry.getValue(), 0.0001);
+    private void assertDownloadStatus(FileDownloader fileDownloader, FileReference fileReference, double expectedDownloadStatus) {
+        double downloadStatus = fileDownloader.downloadStatus(fileReference);
+        assertEquals(expectedDownloadStatus, downloadStatus, 0.0001);
+    }
+
+    private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection {
+
+        private ResponseHandler responseHandler;
+
+        MockConnection() {
+            this(new FileReferenceFoundResponseHandler());
+        }
+
+        MockConnection(ResponseHandler responseHandler) {
+            this.responseHandler = responseHandler;
+        }
+
+        @Override
+        public void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter) {
+            responseHandler.request(request);
+        }
+
+        @Override
+        public void invokeSync(Request request, double jrtTimeout) {
+            responseHandler.request(request);
+        }
+
+        @Override
+        public void setError(int errorCode) {
+        }
+
+        @Override
+        public void setSuccess() {
+        }
+
+        @Override
+        public String getAddress() {
+            return null;
+        }
+
+        @Override
+        public void close() {
+        }
+
+        @Override
+        public void setError(Connection connection, int errorCode) {
+            connection.setError(errorCode);
+        }
+
+        @Override
+        public Connection getCurrent() {
+            return this;
+        }
+
+        @Override
+        public Connection setNewCurrentConnection() {
+            return this;
+        }
+
+        @Override
+        public int getSize() {
+            return 1;
+        }
+
+        public void setResponseHandler(ResponseHandler responseHandler) {
+            this.responseHandler = responseHandler;
+        }
+
+        static class FileReferenceFoundResponseHandler implements ResponseHandler {
+
+            @Override
+            public void request(Request request) {
+                if (request.methodName().equals("filedistribution.serveFile"))
+                    request.returnValues().add(new Int32Value(0));
+            }
+        }
+
+        static class UnknownFileReferenceResponseHandler implements ResponseHandler {
+
+            @Override
+            public void request(Request request) {
+                if (request.methodName().equals("filedistribution.serveFile"))
+                    request.returnValues().add(new Int32Value(1));
+            }
+        }
+
+        public interface ResponseHandler {
+
+            void request(Request request);
+
+        }
+
     }
+
 }