diff options
6 files changed, 130 insertions, 24 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index b1277682849..b228f0ca5c8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.filedistribution.CompressedFileReference; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceData; +import com.yahoo.vespa.filedistribution.FileReferenceDataBlob; import java.io.File; import java.io.IOException; @@ -110,7 +111,7 @@ public class FileServer { log.info("Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); boolean success = false; String errorDescription = "OK"; - FileReferenceData fileData = FileReferenceData.empty(reference, file.getName()); + FileReferenceData fileData = FileReferenceDataBlob.empty(reference, file.getName()); try { fileData = readFileReferenceData(reference); success = true; @@ -138,7 +139,7 @@ public class FileServer { blob = IOUtils.readFileBytes(file); } - return new FileReferenceData(reference, file.getName(), type, blob); + return new FileReferenceDataBlob(reference, file.getName(), type, blob); } public void serveFile(Request request, Receiver receiver) { pullExecutor.execute(() -> serveFile(request.parameters().get(0).asString(), request, receiver)); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 2452a42cf9e..aa2f0310754 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -187,7 +187,7 @@ public class FileReceiver { if (errorCode == 0) { // TODO: Remove when system test works log.log(LogLevel.INFO, "Receiving file reference '" + fileReference.value() + "'"); - receiveFile(new FileReferenceData(fileReference, filename, FileReferenceData.Type.valueOf(type), content, xxhash)); + receiveFile(new FileReferenceDataBlob(fileReference, filename, FileReferenceData.Type.valueOf(type), content, xxhash)); req.returnValues().add(new Int32Value(0)); } else { log.log(LogLevel.WARNING, "Receiving file reference '" + fileReference.value() + "' failed: " + errorDescription); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java index 809e9e8a6a7..171a3d7fb39 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java @@ -2,39 +2,27 @@ package com.yahoo.vespa.filedistribution; import com.yahoo.config.FileReference; -import net.jpountz.xxhash.XXHashFactory; import java.nio.ByteBuffer; + /** * Utility class for a file reference with data and metadata * * @author hmusum */ -public class FileReferenceData { +public abstract class FileReferenceData { public enum Type {file, compressed} private final FileReference fileReference; private final String filename; private final Type type; - private final byte[] content; - private final long xxhash; - - public FileReferenceData(FileReference fileReference, String filename, Type type, byte[] content) { - this(fileReference, filename, type, content, XXHashFactory.fastestInstance().hash64().hash(ByteBuffer.wrap(content), 0)); - } - public FileReferenceData(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) { + public FileReferenceData(FileReference fileReference, String filename, Type type) { this.fileReference = fileReference; this.filename = filename; this.type = type; - this.content = content; - this.xxhash = xxhash; - } - - public static FileReferenceData empty(FileReference fileReference, String filename) { - return new FileReferenceData(fileReference, filename, FileReferenceData.Type.file, new byte[0], 0); } public FileReference fileReference() { @@ -49,11 +37,31 @@ public class FileReferenceData { return type; } - public byte[] content() { - return content; + public byte [] content() { + ByteBuffer bb = ByteBuffer.allocate((int)size()); + for (byte [] part = nextContent(0); part != null && part.length > 0; part = nextContent(0)) { + bb.put(part); + } + return bb.array(); } + /** + * Will provide the next part of the content. + * + * @param desiredSize of the part + * @return The next part of the content. Empty when done. + */ + public abstract byte[] nextContent(int desiredSize); - public long xxhash() { - return xxhash; - } + /** + * Only guaranteed to be valid after all content has been consumed. + * @return xx64hash of content + */ + public abstract long xxhash(); + + /** + * The size of the content in bytes + * + * @return number of bytes + */ + public abstract long size(); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java new file mode 100644 index 00000000000..f31f9941d71 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java @@ -0,0 +1,43 @@ +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import net.jpountz.xxhash.XXHashFactory; + +import java.nio.ByteBuffer; + +public class FileReferenceDataBlob extends FileReferenceData { + private final byte[] content; + private final long xxhash; + + public FileReferenceDataBlob(FileReference fileReference, String filename, Type type, byte[] content) { + this(fileReference, filename, type, content, XXHashFactory.fastestInstance().hash64().hash(ByteBuffer.wrap(content), 0)); + } + + public FileReferenceDataBlob(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) { + super(fileReference, filename, type); + this.content = content; + this.xxhash = xxhash; + } + + public static FileReferenceData empty(FileReference fileReference, String filename) { + return new FileReferenceDataBlob(fileReference, filename, FileReferenceData.Type.file, new byte[0], 0); + } + + public byte [] content() { + return content; + } + @Override + public byte[] nextContent(int desiredSize) { + return content; + } + + @Override + public long xxhash() { + return xxhash; + } + + @Override + public long size() { + return content.length; + } +} diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java new file mode 100644 index 00000000000..1de72eaefd2 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java @@ -0,0 +1,54 @@ +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import net.jpountz.xxhash.StreamingXXHash64; +import net.jpountz.xxhash.XXHashFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; + +public class LazyFileReferenceData extends FileReferenceData { + private final File file; + private final ReadableByteChannel channel; + private final StreamingXXHash64 hasher; + public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException { + super(fileReference, filename, type); + this.file = file; + channel = Files.newByteChannel(file.toPath()); + this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0); + } + + @Override + public byte[] nextContent(int desiredSize) { + ByteBuffer bb = ByteBuffer.allocate(Math.min(desiredSize, 0x100000)); + try { + channel.read(bb); + } catch (IOException e) { + return null; + } + byte [] retval = bb.array(); + if (bb.position() != bb.array().length) { + retval = new byte [bb.position()]; + bb.get(retval); + } + hasher.update(retval, 0, retval.length); + return retval; + } + + @Override + public long xxhash() { + return hasher.getValue(); + } + + @Override + public long size() { + try { + return Files.size(file.toPath()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index 1c9e8cdb91b..4618b229de1 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -222,7 +222,7 @@ public class FileDownloaderTest { } private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, byte[] content) { - fileDownloader.receiveFile(new FileReferenceData(fileReference, filename, type, content)); + fileDownloader.receiveFile(new FileReferenceDataBlob(fileReference, filename, type, content)); } private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { |