Do registration of references early and only use fileReferences when sending.

8 files changed, 51 insertions, 72 deletions

diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java
index 5d71376aa5b..f024f2ac270 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java
@@ -32,7 +32,9 @@ public class MockFileRegistry implements FileRegistry {
 
     @Override
     public FileReference addUri(String uri) {
-        throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName());
+        FileReference fileReference = new FileReference(uri);
+        entries.add(new Entry(uri, fileReference));
+        return fileReference;
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
index 4793caad308..e31d90c2800 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
@@ -1,6 +1,7 @@
 package com.yahoo.searchdefinition;
 
 import com.yahoo.config.FileReference;
+import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.path.Path;
 import com.yahoo.vespa.model.AbstractService;
 import com.yahoo.vespa.model.utils.FileSender;
@@ -16,7 +17,7 @@ public class DistributableResource {
     private final String name;
     private final ByteBuffer blob;
     private String path;
-    private String fileReference = "";
+    private FileReference fileReference = new FileReference("");
     private PathType pathType = PathType.FILE;
 
     public PathType getPathType() {
@@ -54,18 +55,7 @@ public class DistributableResource {
 
     /** Initiate sending of this constant to some services over file distribution */
     public void sendTo(Collection<? extends AbstractService> services) {
-        fileReference = sendToServices(services).value();
-    }
-    private FileReference sendToServices(Collection<? extends AbstractService> services) {
-        switch (pathType) {
-            case FILE:
-                return FileSender.sendFileToServices(path, services);
-            case URI:
-                return FileSender.sendUriToServices(path, services);
-            case BLOB:
-                return FileSender.sendBlobToServices(blob, services);
-        }
-        throw new IllegalArgumentException("Unknown path type " + pathType);
+        FileSender.send(fileReference, services);
     }
 
     public String getName() { return name; }
@@ -73,7 +63,7 @@ public class DistributableResource {
     public String getFileName() { return path; }
     public Path getFilePath() { return Path.fromString(path); }
     public String getUri() { return path; }
-    public String getFileReference() { return fileReference; }
+    public String getFileReference() { return fileReference.value(); }
 
     public void validate() {
         switch (pathType) {
@@ -88,6 +78,22 @@ public class DistributableResource {
         }
     }
 
+    void register(FileRegistry fileRegistry) {
+        switch (pathType) {
+            case FILE:
+                fileReference = fileRegistry.addFile(path);
+                break;
+            case URI:
+                fileReference = fileRegistry.addUri(path);
+                break;
+            case BLOB:
+                fileReference = fileRegistry.addBlob(blob);
+                break;
+            default:
+                throw new IllegalArgumentException("Unknown path type " + pathType);
+        }
+    }
+
     public String toString() {
         StringBuilder b = new StringBuilder();
         b.append("resource '").append(name).append(" of type '").append(pathType)
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java
index 518221aac96..9d8e8b90ceb 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java
@@ -18,6 +18,7 @@ public class LargeRankExpressions {
 
     public void add(RankExpressionBody expression) {
         expression.validate();
+        expression.register(fileRegistry);
         String name = expression.getName();
         if (expressions.containsKey(name)) {
             throw new IllegalArgumentException("Rank expression '" + name +
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java
index 96ee8c0c2f4..cb61b8c9cec 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java
@@ -24,6 +24,7 @@ public class OnnxModels {
     }
     public void add(OnnxModel model) {
         model.validate();
+        model.register(fileRegistry);
         String name = model.getName();
         models.put(name, model);
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java
index a61e1ae3efb..8ccc0ef429e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java
@@ -26,6 +26,7 @@ public class RankingConstants {
 
     public void add(RankingConstant constant) {
         constant.validate();
+        constant.register(fileRegistry);
         String name = constant.getName();
         if (constants.containsKey(name))
             throw new IllegalArgumentException("Ranking constant '" + name + "' defined twice");
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
index ea0452a6c49..4f8518b0c32 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
@@ -8,7 +8,6 @@ import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.vespa.defaults.Defaults;
 
-import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
@@ -400,7 +399,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
     public void setVespaMallocDebugStackTrace(String s) { vespaMallocDebugStackTrace = s; }
 
     public String getMMapNoCoreEnvVariable() {
-        return (getMMapNoCoreLimit() >= 0l)
+        return (getMMapNoCoreLimit() >= 0L)
                 ? "VESPA_MMAP_NOCORE_LIMIT=" + getMMapNoCoreLimit() + " "
                 : "";
     }
@@ -457,9 +456,14 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
     /**
      * Add the given file to the application's file distributor.
      *
-     * @param relativePath path to the file, relative to the app package
-     * @return the file reference hash
+     * @param reference file reference (hash)
      */
+    public void send(FileReference reference) {
+        Host host = null;
+        if (getHost() != null) // false when running application tests without hosts
+            host = getHost().getHost();
+        getRoot().getFileDistributor().sendFileReference(reference, host);
+    }
     public FileReference sendFile(String relativePath) {
         Host host = null;
         if (getHost() != null) // false when running application tests without hosts
@@ -467,14 +471,6 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
         return getRoot().getFileDistributor().sendFileToHost(relativePath, host);
     }
 
-    public FileReference sendUri(String uri) {
-        return getRoot().getFileDistributor().sendUriToHost(uri, getHost().getHost());
-    }
-
-    public FileReference sendBlob(ByteBuffer blob) {
-        return getRoot().getFileDistributor().sendBlobToHost(blob, getHost().getHost());
-    }
-
     /** The service HTTP port for health status */
     public int getHealthPort() { return -1;}
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
index cf2ff2f5c3b..0f1aaca4454 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
@@ -5,7 +5,6 @@ import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.vespa.model.Host;
 
-import java.nio.ByteBuffer;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -41,24 +40,8 @@ public class FileDistributor {
         return addFileReference(fileRegistry.addFile(relativePath), host);
     }
 
-    /**
-     * Adds the given file to the associated application packages' registry of file and marks the file
-     * for distribution to the given host.
-     *
-     * @return the reference to the file, created by the application package
-     */
-    public FileReference sendUriToHost(String uri, Host host) {
-        return addFileReference(fileRegistry.addUri(uri), host);
-    }
-
-    /**
-     * Adds the given blob to the associated application packages' registry of file and marks the file
-     * for distribution to the given host.
-     *
-     * @return the reference to the file, created by the application package
-     */
-    public FileReference sendBlobToHost(ByteBuffer blob, Host host) {
-        return addFileReference(fileRegistry.addBlob(blob), host);
+    public void sendFileReference(FileReference reference, Host host) {
+        addFileReference(reference, host);
     }
 
     private FileReference addFileReference(FileReference reference, Host host) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
index 52edec7114b..b7bb9f10ade 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
@@ -12,7 +12,6 @@ import com.yahoo.vespa.config.ConfigPayloadBuilder;
 import com.yahoo.vespa.model.AbstractService;
 
 import java.io.Serializable;
-import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
@@ -28,50 +27,40 @@ public class FileSender implements Serializable {
     /**
      * Send the given file to all given services.
      *
-     * @param relativePath  The path to the file, relative to the app pkg.
+     * @param fileReference  The file reference to send.
      * @param services  The services to send the file to.
-     * @return The file reference that the file was given, never null.
      * @throws IllegalStateException if services is empty.
      */
-    public static FileReference sendFileToServices(String relativePath,
-                                                   Collection<? extends AbstractService> services) {
-        if (services.isEmpty()) {
-            throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " +
-                                            "using 'count' instead of <node> tags.");
-        }
-
-        FileReference fileref = null;
-        for (AbstractService service : services) {
-            // The same reference will be returned from each call.
-            fileref = service.sendFile(relativePath);
-        }
-        return fileref;
-    }
-
-    public static FileReference sendUriToServices(String uri, Collection<? extends AbstractService> services) {
+    public static void send(FileReference fileReference, Collection<? extends AbstractService> services) {
         if (services.isEmpty()) {
             throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " +
                     "using 'count' instead of <node> tags.");
         }
 
-        FileReference fileref = null;
         for (AbstractService service : services) {
             // The same reference will be returned from each call.
-            fileref = service.sendUri(uri);
+            service.send(fileReference);
         }
-        return fileref;
     }
-
-    public static FileReference sendBlobToServices(ByteBuffer blob, Collection<? extends AbstractService> services) {
+    /**
+     * Send the given file to all given services.
+     *
+     * @param relativePath  The path to the file, relative to the app pkg.
+     * @param services  The services to send the file to.
+     * @return The file reference that the file was given, never null.
+     * @throws IllegalStateException if services is empty.
+     */
+    public static FileReference sendFileToServices(String relativePath,
+                                                   Collection<? extends AbstractService> services) {
         if (services.isEmpty()) {
             throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " +
-                    "using 'count' instead of <node> tags.");
+                                            "using 'count' instead of <node> tags.");
         }
 
         FileReference fileref = null;
         for (AbstractService service : services) {
             // The same reference will be returned from each call.
-            fileref = service.sendBlob(blob);
+            fileref = service.sendFile(relativePath);
         }
         return fileref;
     }