diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2017-12-08 16:45:48 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2017-12-08 16:45:48 +0100 |
commit | 6bffc191b8689e0dac623d0db90790719a896634 (patch) | |
tree | 05debfc60b67988c0df65eb74a3c374bbf716b09 /filedistribution | |
parent | f4a20225fe622e91fed4c220da09634f97229ea3 (diff) |
Make a abstract FileReferenceData so it can cater for both whole files and chunked files.
Diffstat (limited to 'filedistribution')
5 files changed, 127 insertions, 22 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 2452a42cf9e..aa2f0310754 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -187,7 +187,7 @@ public class FileReceiver { if (errorCode == 0) { // TODO: Remove when system test works log.log(LogLevel.INFO, "Receiving file reference '" + fileReference.value() + "'"); - receiveFile(new FileReferenceData(fileReference, filename, FileReferenceData.Type.valueOf(type), content, xxhash)); + receiveFile(new FileReferenceDataBlob(fileReference, filename, FileReferenceData.Type.valueOf(type), content, xxhash)); req.returnValues().add(new Int32Value(0)); } else { log.log(LogLevel.WARNING, "Receiving file reference '" + fileReference.value() + "' failed: " + errorDescription); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java index 809e9e8a6a7..171a3d7fb39 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java @@ -2,39 +2,27 @@ package com.yahoo.vespa.filedistribution; import com.yahoo.config.FileReference; -import net.jpountz.xxhash.XXHashFactory; import java.nio.ByteBuffer; + /** * Utility class for a file reference with data and metadata * * @author hmusum */ -public class FileReferenceData { +public abstract class FileReferenceData { public enum Type {file, compressed} private final FileReference fileReference; private final String filename; private final Type type; - private final byte[] content; - private final long xxhash; - - public FileReferenceData(FileReference fileReference, String filename, Type type, byte[] content) { - this(fileReference, filename, type, content, XXHashFactory.fastestInstance().hash64().hash(ByteBuffer.wrap(content), 0)); - } - public FileReferenceData(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) { + public FileReferenceData(FileReference fileReference, String filename, Type type) { this.fileReference = fileReference; this.filename = filename; this.type = type; - this.content = content; - this.xxhash = xxhash; - } - - public static FileReferenceData empty(FileReference fileReference, String filename) { - return new FileReferenceData(fileReference, filename, FileReferenceData.Type.file, new byte[0], 0); } public FileReference fileReference() { @@ -49,11 +37,31 @@ public class FileReferenceData { return type; } - public byte[] content() { - return content; + public byte [] content() { + ByteBuffer bb = ByteBuffer.allocate((int)size()); + for (byte [] part = nextContent(0); part != null && part.length > 0; part = nextContent(0)) { + bb.put(part); + } + return bb.array(); } + /** + * Will provide the next part of the content. + * + * @param desiredSize of the part + * @return The next part of the content. Empty when done. + */ + public abstract byte[] nextContent(int desiredSize); - public long xxhash() { - return xxhash; - } + /** + * Only guaranteed to be valid after all content has been consumed. + * @return xx64hash of content + */ + public abstract long xxhash(); + + /** + * The size of the content in bytes + * + * @return number of bytes + */ + public abstract long size(); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java new file mode 100644 index 00000000000..f31f9941d71 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java @@ -0,0 +1,43 @@ +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import net.jpountz.xxhash.XXHashFactory; + +import java.nio.ByteBuffer; + +public class FileReferenceDataBlob extends FileReferenceData { + private final byte[] content; + private final long xxhash; + + public FileReferenceDataBlob(FileReference fileReference, String filename, Type type, byte[] content) { + this(fileReference, filename, type, content, XXHashFactory.fastestInstance().hash64().hash(ByteBuffer.wrap(content), 0)); + } + + public FileReferenceDataBlob(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) { + super(fileReference, filename, type); + this.content = content; + this.xxhash = xxhash; + } + + public static FileReferenceData empty(FileReference fileReference, String filename) { + return new FileReferenceDataBlob(fileReference, filename, FileReferenceData.Type.file, new byte[0], 0); + } + + public byte [] content() { + return content; + } + @Override + public byte[] nextContent(int desiredSize) { + return content; + } + + @Override + public long xxhash() { + return xxhash; + } + + @Override + public long size() { + return content.length; + } +} diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java new file mode 100644 index 00000000000..1de72eaefd2 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java @@ -0,0 +1,54 @@ +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import net.jpountz.xxhash.StreamingXXHash64; +import net.jpountz.xxhash.XXHashFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; + +public class LazyFileReferenceData extends FileReferenceData { + private final File file; + private final ReadableByteChannel channel; + private final StreamingXXHash64 hasher; + public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException { + super(fileReference, filename, type); + this.file = file; + channel = Files.newByteChannel(file.toPath()); + this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0); + } + + @Override + public byte[] nextContent(int desiredSize) { + ByteBuffer bb = ByteBuffer.allocate(Math.min(desiredSize, 0x100000)); + try { + channel.read(bb); + } catch (IOException e) { + return null; + } + byte [] retval = bb.array(); + if (bb.position() != bb.array().length) { + retval = new byte [bb.position()]; + bb.get(retval); + } + hasher.update(retval, 0, retval.length); + return retval; + } + + @Override + public long xxhash() { + return hasher.getValue(); + } + + @Override + public long size() { + try { + return Files.size(file.toPath()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index 1c9e8cdb91b..4618b229de1 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -222,7 +222,7 @@ public class FileDownloaderTest { } private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, byte[] content) { - fileDownloader.receiveFile(new FileReferenceData(fileReference, filename, type, content)); + fileDownloader.receiveFile(new FileReferenceDataBlob(fileReference, filename, type, content)); } private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { |