diff options
Diffstat (limited to 'filedistribution')
-rw-r--r-- | filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java | 9 | ||||
-rw-r--r-- | filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java (renamed from filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java) | 14 | ||||
-rw-r--r-- | filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java | 6 | ||||
-rw-r--r-- | filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java | 28 | ||||
-rw-r--r-- | filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java | 5 | ||||
-rw-r--r-- | filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java | 32 |
6 files changed, 69 insertions, 25 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java index 0ed72076549..c160233dd72 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.filedistribution; import com.google.common.io.ByteStreams; @@ -47,13 +47,6 @@ public class CompressedFileReference { .map(Path::toFile).collect(Collectors.toList()), outputFile); } - public static byte[] compress(File directory) throws IOException { - return compress(directory, Files.find(Paths.get(directory.getAbsolutePath()), - recurseDepth, - (p, basicFileAttributes) -> basicFileAttributes.isRegularFile()) - .map(Path::toFile).collect(Collectors.toList())); - } - public static byte[] compress(File baseDir, List<File> inputFiles) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); TarArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(out)); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java index 1b9b7032698..e6e4fdb8598 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java @@ -1,28 +1,24 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.filedistribution; import com.yahoo.config.FileReference; -import net.jpountz.xxhash.XXHashFactory; import java.nio.ByteBuffer; -public class FileReferenceDataBlob extends FileReferenceData { +public class EmptyFileReferenceData extends FileReferenceData { + private final byte[] content; private final long xxhash; private int contentRead = 0; - public FileReferenceDataBlob(FileReference fileReference, String filename, Type type, byte[] content) { - this(fileReference, filename, type, content, XXHashFactory.fastestInstance().hash64().hash(ByteBuffer.wrap(content), 0)); - } - - public FileReferenceDataBlob(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) { + private EmptyFileReferenceData(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) { super(fileReference, filename, type); this.content = content; this.xxhash = xxhash; } public static FileReferenceData empty(FileReference fileReference, String filename) { - return new FileReferenceDataBlob(fileReference, filename, FileReferenceData.Type.file, new byte[0], 0); + return new EmptyFileReferenceData(fileReference, filename, FileReferenceData.Type.file, new byte[0], 0); } public ByteBuffer content() { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java index 0bc8f3b162a..193111c42fb 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.filedistribution; import com.yahoo.config.FileReference; @@ -12,9 +12,11 @@ import java.nio.channels.ReadableByteChannel; import java.nio.file.Files; public class LazyFileReferenceData extends FileReferenceData { - private final File file; + + protected final File file; private final ReadableByteChannel channel; private final StreamingXXHash64 hasher; + public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException { super(fileReference, filename, type); this.file = file; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java new file mode 100644 index 00000000000..3fb33182c41 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java @@ -0,0 +1,28 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +/** + * File reference data stored in a temporary file that will be deleted when {@link #close()} is called. + */ +public class LazyTemporaryStorageFileReferenceData extends LazyFileReferenceData { + + public LazyTemporaryStorageFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException { + super(fileReference, filename, type, file); + } + + public void close() { + try { + super.close(); + Files.delete(file.toPath()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java index 8dda0bcce66..a9ddff655e3 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java @@ -59,8 +59,9 @@ public class FileReceiverTest { writerB.write("2"); writerB.close(); - byte[] data = CompressedFileReference.compress(dirWithFiles); - transferCompressedData(new FileReference("ref"), "a", data); + File tempFile = temporaryFolder.newFile(); + File file = CompressedFileReference.compress(dirWithFiles, tempFile); + transferCompressedData(new FileReference("ref"), "a", IOUtils.readFileBytes(file)); File downloadDir = new File(root, "ref"); assertEquals("1", IOUtils.readFile(new File(downloadDir, "a"))); assertEquals("2", IOUtils.readFile(new File(downloadDir, "b"))); diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java index f12935609b9..d1dac1131e2 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java @@ -2,34 +2,49 @@ package com.yahoo.vespa.filedistribution; import com.yahoo.config.FileReference; +import com.yahoo.io.IOUtils; import com.yahoo.text.Utf8; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.File; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class FileReferenceDataTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test - public void testDataBlob() { + public void testFileReferenceDataWithTempFile() throws IOException { String content = "blob"; + File tempFile = writeTempFile(content); FileReferenceData fileReferenceData = - new FileReferenceDataBlob(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, Utf8.toBytes(content)); + new LazyTemporaryStorageFileReferenceData(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, tempFile); ByteBuffer byteBuffer = ByteBuffer.allocate(100); assertEquals(4, fileReferenceData.nextContent(byteBuffer)); assertEquals(content, Utf8.toString(Arrays.copyOfRange(byteBuffer.array(), 0, 4))); // nextContent() will always return everything for FileReferenceDataBlob, so nothing more should be read assertEquals(-1, fileReferenceData.nextContent(byteBuffer)); + assertTrue(tempFile.exists()); + fileReferenceData.close(); + assertFalse(tempFile.exists()); // temp file should be removed when closing LazyTemporaryStorageFileReferenceData } @Test - public void testLargerDataBlob() { + public void testFileReferenceData() throws IOException { String content = "blobbblubbblabb"; + File file = writeTempFile(content); FileReferenceData fileReferenceData = - new FileReferenceDataBlob(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, Utf8.toBytes(content)); + new LazyFileReferenceData(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, file); ByteBuffer byteBuffer = ByteBuffer.allocate(10); assertEquals(10, fileReferenceData.nextContent(byteBuffer)); assertEquals(content.substring(0,10), Utf8.toString(Arrays.copyOfRange(byteBuffer.array(), 0, 10))); @@ -39,6 +54,15 @@ public class FileReferenceDataTest { // nextContent() will always return everything for FileReferenceDataBlob, so nothing more should be read assertEquals(-1, fileReferenceData.nextContent(byteBuffer)); + assertTrue(file.exists()); + fileReferenceData.close(); + assertTrue(file.exists()); // file should not be removed + } + + private File writeTempFile(String content) throws IOException { + File file = temporaryFolder.newFile(); + IOUtils.writeFile(file, Utf8.toBytes(content)); + return file; } } |