diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2017-12-08 19:50:40 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2017-12-08 19:50:40 +0100 |
commit | f17e602d96701f81e405e3b740c65df95365e9ca (patch) | |
tree | 1a9239b0e806c445e4ff332e361e70db21888cc7 | |
parent | 08548b78355a3c4fbe1a82e2e54d03156a108b01 (diff) |
Use the LazyFileReferenceData to avoid filling memory.
2 files changed, 12 insertions, 9 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index b228f0ca5c8..81f5e62016a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.filedistribution.CompressedFileReference; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDataBlob; +import com.yahoo.vespa.filedistribution.LazyFileReferenceData; import java.io.File; import java.io.IOException; @@ -125,21 +126,16 @@ public class FileServer { log.info("Done serving reference '" + reference.toString() + "' with file '" + file.getAbsolutePath() + "'"); } - private FileReferenceData readFileReferenceData(FileReference reference) throws IOException { File file = root.getFile(reference); - byte[] blob; - FileReferenceData.Type type; if (file.isDirectory()) { - type = FileReferenceData.Type.compressed; - blob = CompressedFileReference.compress(file.getParentFile()); + //TODO Here we should compress to file, but then we have to clean up too. Pending. + byte [] blob = CompressedFileReference.compress(file.getParentFile()); + return new FileReferenceDataBlob(reference, file.getName(), FileReferenceData.Type.compressed, blob); } else { - type = FileReferenceData.Type.file; - blob = IOUtils.readFileBytes(file); + return new LazyFileReferenceData(reference, file.getName(), FileReferenceData.Type.file, file); } - - return new FileReferenceDataBlob(reference, file.getName(), type, blob); } public void serveFile(Request request, Receiver receiver) { pullExecutor.execute(() -> serveFile(request.parameters().get(0).asString(), request, receiver)); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java index 759a859253e..2c08f5a7605 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java @@ -39,6 +39,13 @@ public class CompressedFileReference { return outputFile; } + public static File compress(File directory, File outputFile) throws IOException { + return compress(directory, Files.find(Paths.get(directory.getAbsolutePath()), + recurseDepth, + (p, basicFileAttributes) -> basicFileAttributes.isRegularFile()) + .map(Path::toFile).collect(Collectors.toList()), outputFile); + } + public static byte[] compress(File directory) throws IOException { return compress(directory, Files.find(Paths.get(directory.getAbsolutePath()), recurseDepth, |