Merge pull request #18185 from vespa-engine/balder/add-blob

Add option for adding blobs

14 files changed, 155 insertions, 21 deletions

diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java
index b3d2b061430..5d71376aa5b 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java
@@ -4,7 +4,9 @@ package com.yahoo.config.model.application.provider;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.net.HostName;
+import net.jpountz.xxhash.XXHashFactory;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -33,4 +35,13 @@ public class MockFileRegistry implements FileRegistry {
         throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName());
     }
 
+    @Override
+    public FileReference addBlob(ByteBuffer blob) {
+        long blobHash = XXHashFactory.fastestJavaInstance().hash64().hash(blob, 0);
+        String relativePath = Long.toHexString(blobHash) + ".blob";
+        FileReference fileReference = new FileReference(relativePath);
+        entries.add(new Entry(relativePath, fileReference));
+        return fileReference;
+    }
+
 }
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java
index e779d59ba24..75482ded05d 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.FileRegistry;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.Reader;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -81,6 +82,10 @@ public class PreGeneratedFileRegistry implements FileRegistry {
     public FileReference addUri(String uri) {
         return new FileReference(path2Hash.get(uri));
     }
+    @Override
+    public FileReference addBlob(ByteBuffer blob) {
+        return new FileReference(path2Hash.get(blob));
+    }
 
     @Override
     public String fileSourceHost() {
diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistryTestCase.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistryTestCase.java
index 4b2f5890a4e..7996efaa60e 100644
--- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistryTestCase.java
+++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistryTestCase.java
@@ -1,35 +1,37 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.model.application.provider;
 
-import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.FileRegistry;
 import org.junit.Test;
 
 import java.io.StringReader;
-import java.util.List;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 /**
  * @author Tony Vaagenes
  */
 public class PreGeneratedFileRegistryTestCase {
 
+    private static final String BLOB = "Some blob";
     @Test
     public void importAndExport() {
         FileRegistry fileRegistry = new MockFileRegistry();
         fileRegistry.addFile("1234");
+        fileRegistry.addBlob(ByteBuffer.wrap(BLOB.getBytes(StandardCharsets.UTF_8)));
         String serializedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry);
 
         PreGeneratedFileRegistry importedRegistry = PreGeneratedFileRegistry.importRegistry(new StringReader(serializedRegistry));
 
-        assertEquals(Set.of("1234"), importedRegistry.getPaths());
+        assertEquals(Set.of("1234", "c5674b55c15c9c95.blob"), importedRegistry.getPaths());
 
-        assertEquals(1, importedRegistry.getPaths().size());
+        assertEquals(2, importedRegistry.getPaths().size());
 
         checkConsistentEntry(fileRegistry.export().get(0), importedRegistry);
+        checkConsistentEntry(fileRegistry.export().get(1), importedRegistry);
         assertEquals(fileRegistry.fileSourceHost(), importedRegistry.fileSourceHost());
     }
 
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json
index 9de674ca820..735778f4d46 100644
--- a/config-model-api/abi-spec.json
+++ b/config-model-api/abi-spec.json
@@ -422,6 +422,7 @@
     "methods": [
       "public abstract com.yahoo.config.FileReference addFile(java.lang.String)",
       "public abstract com.yahoo.config.FileReference addUri(java.lang.String)",
+      "public abstract com.yahoo.config.FileReference addBlob(java.nio.ByteBuffer)",
       "public com.yahoo.config.FileReference addApplicationPackage()",
       "public abstract java.lang.String fileSourceHost()",
       "public abstract java.util.List export()"
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java
index 8415781b827..9d049ae0847 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.application.api;
 
+import java.nio.ByteBuffer;
 import java.util.List;
 
 import com.yahoo.config.FileReference;
@@ -13,6 +14,7 @@ public interface FileRegistry {
 
     FileReference addFile(String relativePath);
     FileReference addUri(String uri);
+    FileReference addBlob(ByteBuffer blob);
     default FileReference addApplicationPackage() { return addFile(""); }
 
     /**
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
index 77ce2dd41b5..60431c156fd 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
@@ -5,15 +5,17 @@ import com.yahoo.path.Path;
 import com.yahoo.vespa.model.AbstractService;
 import com.yahoo.vespa.model.utils.FileSender;
 
+import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.Objects;
 
 public class DistributableResource {
-    public enum PathType { FILE, URI };
+    public enum PathType { FILE, URI, BLOB };
 
     /** The search definition-unique name of this constant */
     private final String name;
-    private String path = null;
+    private final ByteBuffer blob;
+    private String path;
     private String fileReference = "";
     private PathType pathType = PathType.FILE;
 
@@ -22,11 +24,20 @@ public class DistributableResource {
     }
 
     public DistributableResource(String name) {
-        this(name, null);
+        this.name = name;
+        blob = null;
     }
     public DistributableResource(String name, String path) {
         this.name = name;
         this.path = path;
+        blob = null;
+    }
+    public DistributableResource(String name, ByteBuffer blob) {
+        Objects.requireNonNull(name, "Blob name cannot be null");
+        Objects.requireNonNull(blob, "Blob cannot be null");
+        this.name = name;
+        this.blob = blob;
+        pathType = PathType.BLOB;
     }
 
     public void setFileName(String fileName) {
@@ -44,13 +55,22 @@ public class DistributableResource {
     protected void setFileReference(String fileReference) { this.fileReference = fileReference; }
     /** Initiate sending of this constant to some services over file distribution */
     public void sendTo(Collection<? extends AbstractService> services) {
-        FileReference reference = (pathType == PathType.FILE)
-                ? FileSender.sendFileToServices(path, services)
-                : FileSender.sendUriToServices(path, services);
-        this.fileReference = reference.value();
+        fileReference = sendToServices(services).value();
+    }
+    private FileReference sendToServices(Collection<? extends AbstractService> services) {
+        switch (pathType) {
+            case FILE:
+                return FileSender.sendFileToServices(path, services);
+            case URI:
+                return FileSender.sendUriToServices(path, services);
+            case BLOB:
+                return FileSender.sendBlobToServices(blob, services);
+        }
+        throw new IllegalArgumentException("Unknown path type " + pathType);
     }
 
     public String getName() { return name; }
+    public ByteBuffer getBlob() { return blob; }
     public String getFileName() { return path; }
     public Path getFilePath() { return Path.fromString(path); }
     public String getUri() { return path; }
@@ -63,10 +83,8 @@ public class DistributableResource {
 
     public String toString() {
         StringBuilder b = new StringBuilder();
-        b.append("resource '").append(name)
-                .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path)
-                .append("' with ref '").append(fileReference)
-                .append("'");
+        b.append("resource '").append(name).append(" of type '").append(pathType)
+                .append("' with ref '").append(fileReference).append("'");
         return b.toString();
     }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
index 8dea1b65079..ea0452a6c49 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
@@ -8,6 +8,7 @@ import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.vespa.defaults.Defaults;
 
+import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
@@ -470,6 +471,10 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
         return getRoot().getFileDistributor().sendUriToHost(uri, getHost().getHost());
     }
 
+    public FileReference sendBlob(ByteBuffer blob) {
+        return getRoot().getFileDistributor().sendBlobToHost(blob, getHost().getHost());
+    }
+
     /** The service HTTP port for health status */
     public int getHealthPort() { return -1;}
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
index 5bb57f4ff6c..d8da911e32f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
@@ -8,6 +8,7 @@ import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.vespa.model.ConfigProxy;
 import com.yahoo.vespa.model.Host;
 
+import java.nio.ByteBuffer;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -58,6 +59,16 @@ public class FileDistributor {
         return addFileReference(fileRegistry.addUri(uri), host);
     }
 
+    /**
+     * Adds the given blob to the associated application packages' registry of file and marks the file
+     * for distribution to the given host.
+     *
+     * @return the reference to the file, created by the application package
+     */
+    public FileReference sendBlobToHost(ByteBuffer blob, Host host) {
+        return addFileReference(fileRegistry.addBlob(blob), host);
+    }
+
     private FileReference addFileReference(FileReference reference, Host host) {
         filesToHosts.computeIfAbsent(reference, k -> new HashSet<>()).add(host);
         return reference;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
index 5e7ac0cabec..52edec7114b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.config.ConfigPayloadBuilder;
 import com.yahoo.vespa.model.AbstractService;
 
 import java.io.Serializable;
+import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
@@ -61,6 +62,20 @@ public class FileSender implements Serializable {
         return fileref;
     }
 
+    public static FileReference sendBlobToServices(ByteBuffer blob, Collection<? extends AbstractService> services) {
+        if (services.isEmpty()) {
+            throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " +
+                    "using 'count' instead of <node> tags.");
+        }
+
+        FileReference fileref = null;
+        for (AbstractService service : services) {
+            // The same reference will be returned from each call.
+            fileref = service.sendBlob(blob);
+        }
+        return fileref;
+    }
+
     /**
      * Sends all user configured files for a producer to all given services.
      */
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
index 2250f2dc579..163c19abe75 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
@@ -3,10 +3,13 @@ package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.config.FileReference;
 
+import java.nio.ByteBuffer;
+
 /**
  * @author baldersheim
  */
 public interface AddFileInterface {
     FileReference addUri(String uri, String relativePath);
     FileReference addFile(String relativePath);
+    FileReference addBlob(ByteBuffer blob, String relativePath);
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
index 4152c92c289..a1907c01085 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
@@ -7,6 +7,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.net.URL;
+import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
 import java.nio.file.Files;
@@ -35,6 +36,32 @@ public class ApplicationFileManager implements AddFileInterface {
         return addFile(relativePath);
     }
 
+    @Override
+    public FileReference addBlob(ByteBuffer blob, String relativePath) {
+        writeBlob(blob, relativePath);
+        return addFile(relativePath);
+    }
+
+    private void writeBlob(ByteBuffer blob, String relativePath) {
+        File file = new File(applicationDir, relativePath);
+        FileOutputStream fos = null;
+        try {
+            Files.createDirectories(file.toPath().getParent());
+            fos = new FileOutputStream(file.getAbsolutePath());
+            fos.write(blob.array(), blob.arrayOffset(), blob.remaining());
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Failed creating directory " + file.getParent(), e);
+        } finally {
+            try {
+                if (fos != null) {
+                    fos.close();
+                }
+            } catch (IOException e) {
+                throw new IllegalArgumentException("Failed closing down after writing blob of size " + blob.remaining() + " to " + file.getAbsolutePath());
+            }
+        }
+    }
+
     private void download(String uri, String relativePath) {
         File file = new File(applicationDir, relativePath);
         FileOutputStream fos = null;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
index ce582a8a1a8..4605d5e5f5c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
@@ -51,6 +51,22 @@ public class FileDBRegistry implements FileRegistry {
     }
 
     @Override
+    public FileReference addBlob(ByteBuffer blob) {
+        long blobHash = XXHashFactory.fastestJavaInstance().hash64().hash(blob, 0);
+        String blobName = Long.toHexString(blobHash);
+        String relativePath = blobToRelativeFile(blob, blobName);
+        synchronized (this) {
+            Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(blobName));
+            return cachedReference.orElseGet(() -> {
+                FileReference newRef = manager.addBlob(blob, relativePath);
+                entries.add(new Entry(blobName, newRef));
+                fileReferenceCache.put(blobName, newRef);
+                return newRef;
+            });
+        }
+    }
+
+    @Override
     public String fileSourceHost() {
         return HostName.getLocalhost();
     }
@@ -72,4 +88,9 @@ public class FileDBRegistry implements FileRegistry {
         return relative;
     }
 
+    private static String blobToRelativeFile(ByteBuffer blob, String blobName) {
+        String relative = "blob/" + blobName;
+        return relative;
+    }
+
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java
index 890a31645fd..723adc1400b 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java
@@ -4,8 +4,10 @@ package com.yahoo.vespa.config.server.filedistribution;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.net.HostName;
+import net.jpountz.xxhash.XXHashFactory;
 
 import java.io.File;
+import java.nio.ByteBuffer;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
@@ -45,4 +47,14 @@ public class MockFileRegistry implements FileRegistry {
         throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName());
     }
 
+    @Override
+    public FileReference addBlob(ByteBuffer blob) {
+        long blobHash = XXHashFactory.fastestJavaInstance().hash64().hash(blob, 0);
+        String relativePath = "./" + Long.toHexString(blobHash) + ".blob";
+        FileReference fileReference = addFileInterface.addBlob(blob, relativePath);
+
+        entries.add(new Entry(relativePath, fileReference));
+        return fileReference;
+    }
+
 }
diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java
index 4d968914dfa..d21211bdffe 100644
--- a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java
+++ b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java
@@ -9,11 +9,11 @@ import com.yahoo.net.HostName;
 import java.io.File;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.nio.ByteBuffer;
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -78,12 +78,13 @@ public class LocalFileDb implements FileAcquirer, FileRegistry {
         throw new RuntimeException("addUri(String uri) is not implemented here.");
     }
 
-    public String fileSourceHost() {
-        return HostName.getLocalhost();
+    @Override
+    public FileReference addBlob(ByteBuffer blob) {
+        throw new RuntimeException("addBlob(ByteBuffer blob) is not implemented here.");
     }
 
-    public Set<String> allRelativePaths() {
-        return fileReferenceToFile.values().stream().map(File::getPath).collect(Collectors.toSet());
+    public String fileSourceHost() {
+        return HostName.getLocalhost();
     }
 
     private static Constructor<FileReference> createFileReferenceConstructor() {