Make a abstract FileReferenceData so it can cater for both whole files and chunked files.

5 files changed, 127 insertions, 22 deletions

diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
index 2452a42cf9e..aa2f0310754 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
@@ -187,7 +187,7 @@ public class FileReceiver {
         if (errorCode == 0) {
             // TODO: Remove when system test works
             log.log(LogLevel.INFO, "Receiving file reference '" + fileReference.value() + "'");
-            receiveFile(new FileReferenceData(fileReference, filename, FileReferenceData.Type.valueOf(type), content, xxhash));
+            receiveFile(new FileReferenceDataBlob(fileReference, filename, FileReferenceData.Type.valueOf(type), content, xxhash));
             req.returnValues().add(new Int32Value(0));
         } else {
             log.log(LogLevel.WARNING, "Receiving file reference '" + fileReference.value() + "' failed: " + errorDescription);
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java
index 809e9e8a6a7..171a3d7fb39 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java
@@ -2,39 +2,27 @@
 package com.yahoo.vespa.filedistribution;
 
 import com.yahoo.config.FileReference;
-import net.jpountz.xxhash.XXHashFactory;
 
 import java.nio.ByteBuffer;
 
+
 /**
  * Utility class for a file reference with data and metadata
  *
  * @author hmusum
  */
-public class FileReferenceData {
+public abstract class FileReferenceData {
 
     public enum Type {file, compressed}
 
     private final FileReference fileReference;
     private final String filename;
     private final Type type;
-    private final byte[] content;
-    private final long xxhash;
-
-    public FileReferenceData(FileReference fileReference, String filename, Type type, byte[] content) {
-        this(fileReference, filename, type, content, XXHashFactory.fastestInstance().hash64().hash(ByteBuffer.wrap(content), 0));
-    }
 
-    public FileReferenceData(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) {
+    public FileReferenceData(FileReference fileReference, String filename, Type type) {
         this.fileReference = fileReference;
         this.filename = filename;
         this.type = type;
-        this.content = content;
-        this.xxhash = xxhash;
-    }
-
-    public static FileReferenceData empty(FileReference fileReference, String filename) {
-        return new FileReferenceData(fileReference, filename, FileReferenceData.Type.file, new byte[0], 0);
     }
 
     public FileReference fileReference() {
@@ -49,11 +37,31 @@ public class FileReferenceData {
         return type;
     }
 
-    public byte[] content() {
-        return content;
+    public byte [] content() {
+        ByteBuffer bb = ByteBuffer.allocate((int)size());
+        for (byte [] part = nextContent(0); part != null && part.length > 0; part = nextContent(0)) {
+            bb.put(part);
+        }
+        return bb.array();
     }
+    /**
+     * Will provide the next part of the content.
+     *
+     * @param desiredSize of the part
+     * @return The next part of the content. Empty when done.
+     */
+    public abstract byte[] nextContent(int desiredSize);
 
-    public long xxhash() {
-        return xxhash;
-    }
+    /**
+     * Only guaranteed to be valid after all content has been consumed.
+     * @return xx64hash of content
+     */
+    public abstract long xxhash();
+
+    /**
+     * The size of the content in bytes
+     *
+     * @return number of bytes
+     */
+    public abstract long size();
 }
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java
new file mode 100644
index 00000000000..f31f9941d71
--- /dev/null
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDataBlob.java
@@ -0,0 +1,43 @@
+package com.yahoo.vespa.filedistribution;
+
+import com.yahoo.config.FileReference;
+import net.jpountz.xxhash.XXHashFactory;
+
+import java.nio.ByteBuffer;
+
+public class FileReferenceDataBlob extends FileReferenceData {
+    private final byte[] content;
+    private final long xxhash;
+
+    public FileReferenceDataBlob(FileReference fileReference, String filename, Type type, byte[] content) {
+        this(fileReference, filename, type, content, XXHashFactory.fastestInstance().hash64().hash(ByteBuffer.wrap(content), 0));
+    }
+
+    public FileReferenceDataBlob(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) {
+        super(fileReference, filename, type);
+        this.content = content;
+        this.xxhash = xxhash;
+    }
+
+    public static FileReferenceData empty(FileReference fileReference, String filename) {
+        return new FileReferenceDataBlob(fileReference, filename, FileReferenceData.Type.file, new byte[0], 0);
+    }
+
+    public byte [] content() {
+        return content;
+    }
+    @Override
+    public byte[] nextContent(int desiredSize) {
+        return content;
+    }
+
+    @Override
+    public long xxhash() {
+        return xxhash;
+    }
+
+    @Override
+    public long size() {
+        return content.length;
+    }
+}
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java
new file mode 100644
index 00000000000..1de72eaefd2
--- /dev/null
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java
@@ -0,0 +1,54 @@
+package com.yahoo.vespa.filedistribution;
+
+import com.yahoo.config.FileReference;
+import net.jpountz.xxhash.StreamingXXHash64;
+import net.jpountz.xxhash.XXHashFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.Files;
+
+public class LazyFileReferenceData extends FileReferenceData {
+    private final File file;
+    private final ReadableByteChannel channel;
+    private final StreamingXXHash64 hasher;
+    public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException {
+        super(fileReference, filename, type);
+        this.file = file;
+        channel = Files.newByteChannel(file.toPath());
+        this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0);
+    }
+
+    @Override
+    public byte[] nextContent(int desiredSize) {
+        ByteBuffer bb = ByteBuffer.allocate(Math.min(desiredSize, 0x100000));
+        try {
+            channel.read(bb);
+        } catch (IOException e) {
+            return null;
+        }
+        byte [] retval = bb.array();
+        if (bb.position() != bb.array().length) {
+            retval = new byte [bb.position()];
+            bb.get(retval);
+        }
+        hasher.update(retval, 0, retval.length);
+        return retval;
+    }
+
+    @Override
+    public long xxhash() {
+        return hasher.getValue();
+    }
+
+    @Override
+    public long size() {
+        try {
+            return Files.size(file.toPath());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
index 1c9e8cdb91b..4618b229de1 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
@@ -222,7 +222,7 @@ public class FileDownloaderTest {
     }
 
     private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, byte[] content) {
-        fileDownloader.receiveFile(new FileReferenceData(fileReference, filename, type, content));
+        fileDownloader.receiveFile(new FileReferenceDataBlob(fileReference, filename, type, content));
     }
 
     private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection {