Write content for files in diretory to file

Instead of storing data from file reference that is a directory in a byte array
(which might be huge and use a lot of memory), store in a temporary file
and delete file when calling close()

2 files changed, 31 insertions, 6 deletions

diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
index 8dda0bcce66..a9ddff655e3 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
@@ -59,8 +59,9 @@ public class FileReceiverTest {
         writerB.write("2");
         writerB.close();
 
-        byte[] data = CompressedFileReference.compress(dirWithFiles);
-        transferCompressedData(new FileReference("ref"), "a", data);
+        File tempFile = temporaryFolder.newFile();
+        File file = CompressedFileReference.compress(dirWithFiles, tempFile);
+        transferCompressedData(new FileReference("ref"), "a", IOUtils.readFileBytes(file));
         File downloadDir = new File(root, "ref");
         assertEquals("1", IOUtils.readFile(new File(downloadDir, "a")));
         assertEquals("2", IOUtils.readFile(new File(downloadDir, "b")));
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java
index f12935609b9..d1dac1131e2 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java
@@ -2,34 +2,49 @@
 package com.yahoo.vespa.filedistribution;
 
 import com.yahoo.config.FileReference;
+import com.yahoo.io.IOUtils;
 import com.yahoo.text.Utf8;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
+import java.io.File;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 public class FileReferenceDataTest {
 
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
     @Test
-    public void testDataBlob() {
+    public void testFileReferenceDataWithTempFile() throws IOException {
         String content = "blob";
+        File tempFile = writeTempFile(content);
         FileReferenceData fileReferenceData =
-                new FileReferenceDataBlob(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, Utf8.toBytes(content));
+                new LazyTemporaryStorageFileReferenceData(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, tempFile);
         ByteBuffer byteBuffer = ByteBuffer.allocate(100);
         assertEquals(4, fileReferenceData.nextContent(byteBuffer));
         assertEquals(content, Utf8.toString(Arrays.copyOfRange(byteBuffer.array(), 0, 4)));
 
         // nextContent() will always return everything for FileReferenceDataBlob, so nothing more should be read
         assertEquals(-1, fileReferenceData.nextContent(byteBuffer));
+        assertTrue(tempFile.exists());
+        fileReferenceData.close();
+        assertFalse(tempFile.exists()); // temp file should be removed when closing LazyTemporaryStorageFileReferenceData
     }
 
     @Test
-    public void testLargerDataBlob() {
+    public void testFileReferenceData() throws IOException {
         String content = "blobbblubbblabb";
+        File file = writeTempFile(content);
         FileReferenceData fileReferenceData =
-                new FileReferenceDataBlob(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, Utf8.toBytes(content));
+                new LazyFileReferenceData(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, file);
         ByteBuffer byteBuffer = ByteBuffer.allocate(10);
         assertEquals(10, fileReferenceData.nextContent(byteBuffer));
         assertEquals(content.substring(0,10), Utf8.toString(Arrays.copyOfRange(byteBuffer.array(), 0, 10)));
@@ -39,6 +54,15 @@ public class FileReferenceDataTest {
 
         // nextContent() will always return everything for FileReferenceDataBlob, so nothing more should be read
         assertEquals(-1, fileReferenceData.nextContent(byteBuffer));
+        assertTrue(file.exists());
+        fileReferenceData.close();
+        assertTrue(file.exists()); // file should not be removed
+    }
+
+    private File writeTempFile(String content) throws IOException {
+        File file = temporaryFolder.newFile();
+        IOUtils.writeFile(file, Utf8.toBytes(content));
+        return file;
     }
 
 }