Validate xxhash when receiving file

4 files changed, 27 insertions, 9 deletions

diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java
index f3c694f31ab..ac7555c7905 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java
@@ -66,8 +66,8 @@ public class FileDownloader {
         fileReferences.forEach(this::queueForDownload);
     }
 
-    public void receiveFile(FileReference fileReference, String filename, byte[] content) {
-        fileReferenceDownloader.receiveFile(fileReference, filename, content);
+    public void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) {
+        fileReferenceDownloader.receiveFile(fileReference, filename, content, xxHash);
     }
 
     double downloadStatus(FileReference fileReference) {
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReceiver.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReceiver.java
index 314c90c5853..faf0e186fb3 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReceiver.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReceiver.java
@@ -8,9 +8,12 @@ import com.yahoo.jrt.Method;
 import com.yahoo.jrt.Request;
 import com.yahoo.jrt.Supervisor;
 import com.yahoo.log.LogLevel;
+import net.jpountz.xxhash.XXHash64;
+import net.jpountz.xxhash.XXHashFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.util.logging.Logger;
 
@@ -21,6 +24,7 @@ public class FileReceiver {
     private final Supervisor supervisor;
     private final FileReferenceDownloader downloader;
     private final File downloadDirectory;
+    private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
 
     public FileReceiver(Supervisor supervisor, FileReferenceDownloader downloader, File downloadDirectory) {
         this.supervisor = supervisor;
@@ -60,7 +64,7 @@ public class FileReceiver {
         if (errorCode == 0) {
             // TODO: Remove when system test works
             log.log(LogLevel.INFO, "Receiving file reference '" + fileReference.value() + "'");
-            receiveFile(fileReference, filename, content);
+            receiveFile(fileReference, filename, content, xxhash);
             req.returnValues().add(new Int32Value(0));
         } else {
             log.log(LogLevel.WARNING, "Receiving file reference '" + fileReference.value() + "' failed: " + errorDescription);
@@ -69,7 +73,11 @@ public class FileReceiver {
         }
     }
 
-    void receiveFile(FileReference fileReference, String filename, byte[] content) {
+    void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) {
+        long xxHashFromContent = hasher.hash(ByteBuffer.wrap(content), 0);
+        if (xxHashFromContent != xxHash)
+            throw new RuntimeException("xxhash from content (" + xxHashFromContent + ") is not equal to xxhash in request (" + xxHash + ")");
+
         File fileReferenceDir = new File(downloadDirectory, fileReference.value());
         try {
             Files.createDirectories(fileReferenceDir.toPath());
@@ -83,5 +91,4 @@ public class FileReceiver {
         }
     }
 
-
 }
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java
index eaa791bec29..c972cfbbf56 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java
@@ -79,8 +79,8 @@ class FileReferenceDownloader {
         downloadQueue.add(fileReferenceDownload);
     }
 
-    void receiveFile(FileReference fileReference, String filename, byte[] content) {
-        fileReceiver.receiveFile(fileReference, filename, content);
+    void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) {
+        fileReceiver.receiveFile(fileReference, filename, content, xxHash);
     }
 
     synchronized Set<FileReference> queuedDownloads() {
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java
index 0959919a46d..d1b691b9d5e 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java
@@ -11,11 +11,14 @@ import com.yahoo.jrt.Transport;
 import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.Connection;
 import com.yahoo.vespa.config.ConnectionPool;
+import net.jpountz.xxhash.XXHash64;
+import net.jpountz.xxhash.XXHashFactory;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.time.Duration;
 import java.util.Arrays;
@@ -29,6 +32,8 @@ import static org.junit.Assert.fail;
 
 public class FileDownloaderTest {
 
+    private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
+
     private MockConnection connection;
     private FileDownloader fileDownloader;
     private File downloadDir;
@@ -94,7 +99,7 @@ public class FileDownloaderTest {
 
             // Receives fileReference, should return and make it available to caller
             String filename = "abc.jar";
-            fileDownloader.receiveFile(fileReference, filename, Utf8.toBytes("some other content"));
+            receiveFile(fileReference, filename, "some other content");
             Optional<File> downloadedFile = fileDownloader.getFile(fileReference);
 
             assertTrue(downloadedFile.isPresent());
@@ -128,7 +133,7 @@ public class FileDownloaderTest {
     public void receiveFile() throws IOException {
         FileReference foo = new FileReference("foo");
         String filename = "foo.jar";
-        fileDownloader.receiveFile(foo, filename, Utf8.toBytes("content"));
+        receiveFile(foo, filename, "content");
         File downloadedFile = new File(fileReferenceFullPath(downloadDir, foo), filename);
         assertEquals("content", IOUtils.readFile(downloadedFile));
     }
@@ -147,6 +152,12 @@ public class FileDownloaderTest {
         assertEquals(expectedDownloadStatus, downloadStatus, 0.0001);
     }
 
+    private void receiveFile(FileReference fileReference, String filename, String content) {
+        byte[] contentBytes = Utf8.toBytes(content);
+        long xxHashFromContent = hasher.hash(ByteBuffer.wrap(contentBytes), 0);
+        fileDownloader.receiveFile(fileReference, filename, contentBytes, xxHashFromContent);
+    }
+
     private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection {
 
         private ResponseHandler responseHandler;