diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-06-09 15:01:02 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-06-09 18:28:05 +0200 |
commit | c6f31c85c466234df03228d997522860214a4501 (patch) | |
tree | 855b466f7293e97ff59cd9baf6f871caf915dd9d /configserver/src | |
parent | 18956afb738ba5f72631b880c7af9a16acee7589 (diff) |
Add option for adding blobs
Diffstat (limited to 'configserver/src')
4 files changed, 63 insertions, 0 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java index 2250f2dc579..163c19abe75 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java @@ -3,10 +3,13 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; +import java.nio.ByteBuffer; + /** * @author baldersheim */ public interface AddFileInterface { FileReference addUri(String uri, String relativePath); FileReference addFile(String relativePath); + FileReference addBlob(ByteBuffer blob, String relativePath); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java index 4152c92c289..a1907c01085 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java @@ -7,6 +7,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.net.SocketTimeoutException; import java.net.URL; +import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.nio.file.Files; @@ -35,6 +36,32 @@ public class ApplicationFileManager implements AddFileInterface { return addFile(relativePath); } + @Override + public FileReference addBlob(ByteBuffer blob, String relativePath) { + writeBlob(blob, relativePath); + return addFile(relativePath); + } + + private void writeBlob(ByteBuffer blob, String relativePath) { + File file = new File(applicationDir, relativePath); + FileOutputStream fos = null; + try { + Files.createDirectories(file.toPath().getParent()); + fos = new FileOutputStream(file.getAbsolutePath()); + fos.write(blob.array(), blob.arrayOffset(), blob.remaining()); + } catch (IOException e) { + throw new IllegalArgumentException("Failed creating directory " + file.getParent(), e); + } finally { + try { + if (fos != null) { + fos.close(); + } + } catch (IOException e) { + throw new IllegalArgumentException("Failed closing down after writing blob of size " + blob.remaining() + " to " + file.getAbsolutePath()); + } + } + } + private void download(String uri, String relativePath) { File file = new File(applicationDir, relativePath); FileOutputStream fos = null; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index ce582a8a1a8..4605d5e5f5c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -51,6 +51,22 @@ public class FileDBRegistry implements FileRegistry { } @Override + public FileReference addBlob(ByteBuffer blob) { + long blobHash = XXHashFactory.fastestJavaInstance().hash64().hash(blob, 0); + String blobName = Long.toHexString(blobHash); + String relativePath = blobToRelativeFile(blob, blobName); + synchronized (this) { + Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(blobName)); + return cachedReference.orElseGet(() -> { + FileReference newRef = manager.addBlob(blob, relativePath); + entries.add(new Entry(blobName, newRef)); + fileReferenceCache.put(blobName, newRef); + return newRef; + }); + } + } + + @Override public String fileSourceHost() { return HostName.getLocalhost(); } @@ -72,4 +88,9 @@ public class FileDBRegistry implements FileRegistry { return relative; } + private static String blobToRelativeFile(ByteBuffer blob, String blobName) { + String relative = "blob/" + blobName; + return relative; + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java index 890a31645fd..723adc1400b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java @@ -4,8 +4,10 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.net.HostName; +import net.jpountz.xxhash.XXHashFactory; import java.io.File; +import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -45,4 +47,14 @@ public class MockFileRegistry implements FileRegistry { throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName()); } + @Override + public FileReference addBlob(ByteBuffer blob) { + long blobHash = XXHashFactory.fastestJavaInstance().hash64().hash(blob, 0); + String relativePath = "./" + Long.toHexString(blobHash) + ".blob"; + FileReference fileReference = addFileInterface.addBlob(blob, relativePath); + + entries.add(new Entry(relativePath, fileReference)); + return fileReference; + } + } |