Handle that a file reference is a directory with many files

Compress on the fly if asked for such a file reference

5 files changed, 139 insertions, 44 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
index d8a560c6159..e991341b616 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
@@ -35,7 +35,7 @@ public class FileDirectory  {
         try {
             ensureRootExist();
         } catch (IllegalArgumentException e) {
-            log.warning("Failed creating directory in constructor, will retry on demand : " + e.toString());
+            log.log(LogLevel.WARNING, "Failed creating directory in constructor, will retry on demand : " + e.toString());
         }
     }
 
@@ -70,12 +70,8 @@ public class FileDirectory  {
             throw new IllegalArgumentException("File reference '" + reference.toString() + "' with absolute path '" + dir.getAbsolutePath() + "' is not a directory.");
         }
         File [] files = dir.listFiles(new Filter());
-        if (files.length != 1) {
-            StringBuilder msg = new StringBuilder();
-            for (File f: files) {
-                msg.append(f.getName()).append("\n");
-            }
-            throw new IllegalArgumentException("File reference '" + reference.toString() + "' with absolute path '" + dir.getAbsolutePath() + " does not contain exactly one file, but [" + msg.toString() + "]");
+        if (files == null || files.length == 0) {
+            throw new IllegalArgumentException("File reference '" + reference.toString() + "' with absolute path '" + dir.getAbsolutePath() + " does not contain any files");
         }
         return files[0];
     }
@@ -96,25 +92,28 @@ public class FileDirectory  {
         }
     }
 
-    public FileReference addFile(File source, FileReference reference) {
+    FileReference addFile(File source, FileReference reference) {
         ensureRootExist();
         try {
             logfileInfo(source);
             File destinationDir = new File(root, reference.value());
+            Path tempDestinationDir = Files.createTempDirectory(root.toPath(), "writing");
+            File destination = new File(tempDestinationDir.toFile(), source.getName());
             if (!destinationDir.exists()) {
                 destinationDir.mkdir();
-                Path tempDestinationDir = Files.createTempDirectory(root.toPath(), "writing");
-                File destination = new File(tempDestinationDir.toFile(), source.getName());
-                if (source.isDirectory())
-                    IOUtils.copyDirectory(source, destination);
-                else
+                log.log(LogLevel.DEBUG, "file reference ' " + reference.value() + "', source: " + source.getAbsolutePath() );
+                if (source.isDirectory()) {
+                    log.log(LogLevel.DEBUG, "Copying source " + source.getAbsolutePath() + " to " + destination.getAbsolutePath());
+                    IOUtils.copyDirectory(source, destination, -1);
+                } else
                     copyFile(source, destination);
                 if (!destinationDir.exists()) {
+                    log.log(LogLevel.DEBUG, "Moving from " + tempDestinationDir + " to " + destinationDir.getAbsolutePath());
                     if ( ! tempDestinationDir.toFile().renameTo(destinationDir)) {
-                        log.warning("Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'.");
+                        log.log(LogLevel.WARNING, "Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'.");
                     }
                 } else {
-                    IOUtils.copyDirectory(tempDestinationDir.toFile(), destinationDir, 1);
+                    IOUtils.copyDirectory(tempDestinationDir.toFile(), destinationDir, -1);
                 }
                 IOUtils.recursiveDeleteDir(tempDestinationDir.toFile());
             }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
index 9316a9a5c8e..958f26632ef 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
@@ -14,7 +14,9 @@ import com.yahoo.vespa.config.Connection;
 import com.yahoo.vespa.config.ConnectionPool;
 import com.yahoo.vespa.config.JRTConnectionPool;
 import com.yahoo.vespa.config.server.ConfigServerSpec;
+import com.yahoo.vespa.filedistribution.CompressedFileReference;
 import com.yahoo.vespa.filedistribution.FileDownloader;
+import com.yahoo.vespa.filedistribution.FileReferenceData;
 
 import java.io.File;
 import java.io.IOException;
@@ -26,6 +28,7 @@ import java.util.stream.Collectors;
 
 public class FileServer {
     private static final Logger log = Logger.getLogger(FileServer.class.getName());
+
     private final FileDirectory root;
     private final ExecutorService executor;
     private final FileDownloader downloader;
@@ -43,7 +46,7 @@ public class FileServer {
     }
 
     public interface Receiver {
-        void receive(FileReference reference, String filename, byte [] content, ReplayStatus status);
+        void receive(FileReferenceData fileData, ReplayStatus status);
     }
 
     @Inject
@@ -87,22 +90,39 @@ public class FileServer {
         File file = root.getFile(reference);
         // TODO remove once verified in system tests.
         log.info("Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'");
-        byte [] blob = new byte [0];
         boolean success = false;
         String errorDescription = "OK";
+        FileReferenceData fileData = FileReferenceData.empty(reference, file.getName());
         try {
-            blob = IOUtils.readFileBytes(file);
+            fileData = readFileReferenceData(reference);
             success = true;
         } catch (IOException e) {
-            errorDescription = "For file reference '" + reference.value() + "' I failed reading file '" + file.getAbsolutePath() + "'";
-            log.warning(errorDescription + "for sending to '" + target.toString() + "'. " + e.toString());
+            errorDescription = "For file reference '" + reference.value() + "': failed reading file '" + file.getAbsolutePath() + "'";
+            log.warning(errorDescription + " for sending to '" + target.toString() + "'. " + e.toString());
         }
-        target.receive(reference, file.getName(), blob,
-                new ReplayStatus(success ? 0 : 1, success ? "OK" : errorDescription));
+
+        target.receive(fileData, new ReplayStatus(success ? 0 : 1, success ? "OK" : errorDescription));
         // TODO remove once verified in system tests.
         log.info("Done serving reference '" + reference.toString() + "' with file '" + file.getAbsolutePath() + "'");
     }
 
+
+    private FileReferenceData readFileReferenceData(FileReference reference) throws IOException {
+        File file = root.getFile(reference);
+
+        byte[] blob;
+        FileReferenceData.Type type;
+        if (file.isDirectory()) {
+            type = FileReferenceData.Type.compressed;
+            blob = CompressedFileReference.compress(file.getParentFile());
+        } else {
+            type = FileReferenceData.Type.file;
+            blob = IOUtils.readFileBytes(file);
+        }
+
+        return new FileReferenceData(reference, file.getName(), type, blob);
+    }
+
     public void download(FileReference fileReference) {
         downloader.getFile(fileReference);
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
index d17cdf722ea..5c50fbfc31b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
@@ -41,10 +41,8 @@ import com.yahoo.vespa.config.server.monitoring.MetricUpdaterFactory;
 import com.yahoo.vespa.config.server.tenant.TenantHandlerProvider;
 import com.yahoo.vespa.config.server.tenant.TenantListener;
 import com.yahoo.vespa.config.server.tenant.Tenants;
-import net.jpountz.xxhash.XXHash64;
-import net.jpountz.xxhash.XXHashFactory;
+import com.yahoo.vespa.filedistribution.FileReferenceData;
 
-import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -438,19 +436,19 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
         }
 
         @Override
-        public void receive(FileReference reference, String filename, byte [] content, FileServer.ReplayStatus status) {
-            XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
+        public void receive(FileReferenceData fileData, FileServer.ReplayStatus status) {
             Request fileBlob = new Request("filedistribution.receiveFile");
-            fileBlob.parameters().add(new StringValue(reference.value()));
-            fileBlob.parameters().add(new StringValue(filename));
-            fileBlob.parameters().add(new DataValue(content));
-            fileBlob.parameters().add(new Int64Value(hasher.hash(ByteBuffer.wrap(content), 0)));
+            fileBlob.parameters().add(new StringValue(fileData.fileReference().value()));
+            fileBlob.parameters().add(new StringValue(fileData.filename()));
+            fileBlob.parameters().add(new StringValue(fileData.type().name()));
+            fileBlob.parameters().add(new DataValue(fileData.content()));
+            fileBlob.parameters().add(new Int64Value(fileData.xxhash()));
             fileBlob.parameters().add(new Int32Value(status.getCode()));
             fileBlob.parameters().add(new StringValue(status.getDescription()));
             target.invokeSync(fileBlob, 600);
             if (fileBlob.isError()) {
-                log.warning("Failed delivering reference '" + reference.value() + "' with file '" + filename + "' to " +
-                            target.toString() + " with error : '" + fileBlob.errorMessage() + "'.");
+                log.warning("Failed delivering reference '" + fileData.fileReference().value() + "' with file '" + fileData.filename() + "' to " +
+                            target.toString() + " with error: '" + fileBlob.errorMessage() + "'.");
             }
         }
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java
new file mode 100644
index 00000000000..ad807f9527f
--- /dev/null
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java
@@ -0,0 +1,69 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.config.server.filedistribution;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.io.IOUtils;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.junit.Assert.assertTrue;
+
+public class FileDirectoryTest {
+
+    private FileDirectory fileDirectory;
+
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    @Before
+    public void setup() {
+        fileDirectory = new FileDirectory(temporaryFolder.getRoot());
+    }
+
+    @Test
+    public void requireThatFileReferenceWithFilesWorks() throws IOException {
+        FileReference foo = createFile("foo");
+        FileReference bar = createFile("bar");
+
+        assertTrue(fileDirectory.getFile(foo).exists());
+        assertTrue(fileDirectory.getFile(bar).exists());
+    }
+
+
+    @Test
+    public void requireThatFileReferenceWithSubDirectoriesWorks() throws IOException {
+        FileDirectory fileDirectory = new FileDirectory(temporaryFolder.getRoot());
+
+        FileReference foo = createFileInSubDir("subdir", "foo");
+        FileReference bar = createFileInSubDir("subdir", "bar");
+
+        assertTrue(fileDirectory.getFile(foo).exists());
+        assertTrue(fileDirectory.getFile(bar).exists());
+    }
+
+    // Content in created file is equal to the filename string
+    private FileReference createFile(String filename) throws IOException {
+        File file = temporaryFolder.newFile(filename);
+        IOUtils.writeFile(file, filename, false);
+        return fileDirectory.addFile(file);
+    }
+
+    private FileReference createFileInSubDir(String subdirName, String filename) throws IOException {
+        File subDirectory = new File(temporaryFolder.getRoot(), subdirName);
+        if (!subDirectory.exists())
+            subDirectory.mkdirs();
+        File file = new File(subDirectory, filename);
+        IOUtils.writeFile(file, filename, false);
+        return fileDirectory.addFile(file);
+    }
+
+
+}
+
+
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
index 09260987ac0..5fcaee6e590 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
@@ -2,9 +2,9 @@
 package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
-import com.yahoo.config.FileReference;
 import com.yahoo.io.IOUtils;
 import com.yahoo.net.HostName;
+import com.yahoo.vespa.filedistribution.FileReferenceData;
 import org.junit.Test;
 
 import java.io.File;
@@ -35,7 +35,7 @@ public class FileServerTest {
     }
 
     @Test
-    public void requireThatExistingFileCanbeFound() throws IOException {
+    public void requireThatExistingFileCanBeFound() throws IOException {
         createCleanDir("123");
         IOUtils.writeFile("123/f1", "test", true);
         assertTrue(fs.hasFile("123"));
@@ -50,15 +50,13 @@ public class FileServerTest {
         cleanup();
     }
 
-    private static class FileReceiver implements FileServer.Receiver {
-        CompletableFuture<byte []> content;
-        FileReceiver(CompletableFuture<byte []> content) {
-            this.content = content;
-        }
-        @Override
-        public void receive(FileReference reference, String filename, byte[] content, FileServer.ReplayStatus status) {
-            this.content.complete(content);
-        }
+    @Test
+    public void requireThatFileReferenceWithDirectoryCanBeFound() throws IOException {
+        createCleanDir("124/subdir");
+        IOUtils.writeFile("124/subdir/f1", "test", false);
+        IOUtils.writeFile("124/subdir/f2", "test", false);
+        assertTrue(fs.hasFile("124/subdir"));
+        cleanup();
     }
 
     @Test
@@ -98,6 +96,17 @@ public class FileServerTest {
         assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize());
     }
 
+    private static class FileReceiver implements FileServer.Receiver {
+        CompletableFuture<byte []> content;
+        FileReceiver(CompletableFuture<byte []> content) {
+            this.content = content;
+        }
+        @Override
+        public void receive(FileReferenceData fileData, FileServer.ReplayStatus status) {
+            this.content.complete(fileData.content());
+        }
+    }
+
     private void cleanup() {
         created.forEach((file) -> IOUtils.recursiveDeleteDir(file));
         created.clear();