diff options
author | Harald Musum <musum@verizonmedia.com> | 2022-05-25 14:07:24 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-25 14:07:24 +0200 |
commit | 353df3dd7e0f92a60e94a61093c242c2056cd981 (patch) | |
tree | 3ed42e1d2c4715dc4b938d1f6611bafa0bd89585 /filedistribution/src/main | |
parent | 69bb4e0e94cd71aec2b6b20ba146e40394c8a087 (diff) |
Revert "Support compressing files (not just directories) in file distribution [run-systemtest]"
Diffstat (limited to 'filedistribution/src/main')
3 files changed, 23 insertions, 43 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index da52caefb7d..81a8944149c 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -22,6 +22,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type; + /** * When asking for a file reference, this handles RPC callbacks from config server with file data and metadata. * Uses the same Supervisor as the original caller that requests files, so communication uses the same @@ -107,38 +109,30 @@ public class FileReceiver { verifyHash(hash); File file = new File(fileReferenceDir, fileName); + File decompressedDir = null; try { - switch (fileType) { - case file: - log.log(Level.FINE, () -> "Uncompressed file reference " + fileName + ", storing as " + file.getAbsolutePath()); + if (fileType == Type.file) { + try { Files.createDirectories(fileReferenceDir.toPath()); - moveFileToDestination(inprogressFile, file); - break; - case compressed: - log.log(Level.FINE, () -> "Compressed file reference (directory)" + fileName + ", storing in " + fileReferenceDir.getAbsolutePath()); - decompress(fileType, fileReferenceDir, inprogressFile); - break; - default: - throw new RuntimeException("Unknown file type " + fileType); + } catch (IOException e) { + log.log(Level.SEVERE, "Failed creating directory (" + fileReferenceDir.toPath() + "): " + e.getMessage(), e); + throw new RuntimeException("Failed creating directory (" + fileReferenceDir.toPath() + "): ", e); + } + log.log(Level.FINE, () -> "Uncompressed file, moving to " + file.getAbsolutePath()); + moveFileToDestination(inprogressFile, file); + } else { + decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); + new FileReferenceCompressor(fileType).decompress(inprogressFile, decompressedDir); + moveFileToDestination(decompressedDir, fileReferenceDir); } } catch (IOException e) { log.log(Level.SEVERE, "Failed writing file: " + e.getMessage(), e); throw new RuntimeException("Failed writing file: ", e); } finally { deletePath(inprogressFile); - } - return file; - } - - void decompress(FileReferenceData.Type fileType, File fileReferenceDir, File file) throws IOException { - File decompressedDir = null; - try { - decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); - new FileReferenceCompressor(fileType).decompress(file, decompressedDir); - moveFileToDestination(decompressedDir, fileReferenceDir); - } finally { deletePath(decompressedDir); } + return file; } double percentageReceived() { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index d0957297f79..c36bcd22606 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -47,16 +47,13 @@ public class FileReferenceCompressor { return outputFile; } - public File compress(File file, File outputFile) throws IOException { - if (file.isDirectory()) - return compress(file, - Files.find(Paths.get(file.getAbsolutePath()), - recurseDepth, - (p, basicFileAttributes) -> basicFileAttributes.isRegularFile()) - .map(Path::toFile).collect(Collectors.toList()), - outputFile); - else - return compress(file.getParentFile(), List.of(file), outputFile); + public File compress(File directory, File outputFile) throws IOException { + return compress(directory, + Files.find(Paths.get(directory.getAbsolutePath()), + recurseDepth, + (p, basicFileAttributes) -> basicFileAttributes.isRegularFile()) + .map(Path::toFile).collect(Collectors.toList()), + outputFile); } public void decompress(File inputFile, File outputDir) throws IOException { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java index f1ac8edaabb..03f8d184f94 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java @@ -64,17 +64,6 @@ public abstract class FileReferenceData { */ public abstract void close(); - public static Type from(String type) { - switch (type) { - case "none": - return Type.file; - case "gzip": - return Type.compressed; - default: - throw new RuntimeException("Unknown compression type " + type); - } - } - @Override public String toString() { return fileReference.value() + " (" + filename + "), " + type.name(); } |