diff options
author | Harald Musum <musum@oath.com> | 2017-11-14 14:13:49 +0100 |
---|---|---|
committer | Harald Musum <musum@oath.com> | 2017-11-14 14:13:49 +0100 |
commit | d0c82374457c41da55042a1b11ac5c768da6cc63 (patch) | |
tree | 9654eca823ece2ae853fc710537c2cadbea2f913 /config-proxy/src | |
parent | 3aefb034de5176d6225140312096adc37f520d93 (diff) |
Validate xxhash when receiving file
Diffstat (limited to 'config-proxy/src')
4 files changed, 27 insertions, 9 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java index f3c694f31ab..ac7555c7905 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java @@ -66,8 +66,8 @@ public class FileDownloader { fileReferences.forEach(this::queueForDownload); } - public void receiveFile(FileReference fileReference, String filename, byte[] content) { - fileReferenceDownloader.receiveFile(fileReference, filename, content); + public void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) { + fileReferenceDownloader.receiveFile(fileReference, filename, content, xxHash); } double downloadStatus(FileReference fileReference) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReceiver.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReceiver.java index 314c90c5853..faf0e186fb3 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReceiver.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReceiver.java @@ -8,9 +8,12 @@ import com.yahoo.jrt.Method; import com.yahoo.jrt.Request; import com.yahoo.jrt.Supervisor; import com.yahoo.log.LogLevel; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.logging.Logger; @@ -21,6 +24,7 @@ public class FileReceiver { private final Supervisor supervisor; private final FileReferenceDownloader downloader; private final File downloadDirectory; + private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); public FileReceiver(Supervisor supervisor, FileReferenceDownloader downloader, File downloadDirectory) { this.supervisor = supervisor; @@ -60,7 +64,7 @@ public class FileReceiver { if (errorCode == 0) { // TODO: Remove when system test works log.log(LogLevel.INFO, "Receiving file reference '" + fileReference.value() + "'"); - receiveFile(fileReference, filename, content); + receiveFile(fileReference, filename, content, xxhash); req.returnValues().add(new Int32Value(0)); } else { log.log(LogLevel.WARNING, "Receiving file reference '" + fileReference.value() + "' failed: " + errorDescription); @@ -69,7 +73,11 @@ public class FileReceiver { } } - void receiveFile(FileReference fileReference, String filename, byte[] content) { + void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) { + long xxHashFromContent = hasher.hash(ByteBuffer.wrap(content), 0); + if (xxHashFromContent != xxHash) + throw new RuntimeException("xxhash from content (" + xxHashFromContent + ") is not equal to xxhash in request (" + xxHash + ")"); + File fileReferenceDir = new File(downloadDirectory, fileReference.value()); try { Files.createDirectories(fileReferenceDir.toPath()); @@ -83,5 +91,4 @@ public class FileReceiver { } } - } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java index eaa791bec29..c972cfbbf56 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java @@ -79,8 +79,8 @@ class FileReferenceDownloader { downloadQueue.add(fileReferenceDownload); } - void receiveFile(FileReference fileReference, String filename, byte[] content) { - fileReceiver.receiveFile(fileReference, filename, content); + void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) { + fileReceiver.receiveFile(fileReference, filename, content, xxHash); } synchronized Set<FileReference> queuedDownloads() { diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java index 0959919a46d..d1b691b9d5e 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java @@ -11,11 +11,14 @@ import com.yahoo.jrt.Transport; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; import org.junit.Before; import org.junit.Test; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.time.Duration; import java.util.Arrays; @@ -29,6 +32,8 @@ import static org.junit.Assert.fail; public class FileDownloaderTest { + private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + private MockConnection connection; private FileDownloader fileDownloader; private File downloadDir; @@ -94,7 +99,7 @@ public class FileDownloaderTest { // Receives fileReference, should return and make it available to caller String filename = "abc.jar"; - fileDownloader.receiveFile(fileReference, filename, Utf8.toBytes("some other content")); + receiveFile(fileReference, filename, "some other content"); Optional<File> downloadedFile = fileDownloader.getFile(fileReference); assertTrue(downloadedFile.isPresent()); @@ -128,7 +133,7 @@ public class FileDownloaderTest { public void receiveFile() throws IOException { FileReference foo = new FileReference("foo"); String filename = "foo.jar"; - fileDownloader.receiveFile(foo, filename, Utf8.toBytes("content")); + receiveFile(foo, filename, "content"); File downloadedFile = new File(fileReferenceFullPath(downloadDir, foo), filename); assertEquals("content", IOUtils.readFile(downloadedFile)); } @@ -147,6 +152,12 @@ public class FileDownloaderTest { assertEquals(expectedDownloadStatus, downloadStatus, 0.0001); } + private void receiveFile(FileReference fileReference, String filename, String content) { + byte[] contentBytes = Utf8.toBytes(content); + long xxHashFromContent = hasher.hash(ByteBuffer.wrap(contentBytes), 0); + fileDownloader.receiveFile(fileReference, filename, contentBytes, xxHashFromContent); + } + private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { private ResponseHandler responseHandler; |