diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-08-17 08:36:42 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-08-17 08:36:42 +0200 |
commit | a35d40e01254be402972eecc28bfb43fc2081f20 (patch) | |
tree | 2b72cd003dd28ba68be2f10efbd595ee2895beae | |
parent | ecf3f1cfbd977e924e7152e72b46bf45054300be (diff) |
Do registration of references early and only use fileReferences when sending.
8 files changed, 51 insertions, 72 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java index 5d71376aa5b..f024f2ac270 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java @@ -32,7 +32,9 @@ public class MockFileRegistry implements FileRegistry { @Override public FileReference addUri(String uri) { - throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName()); + FileReference fileReference = new FileReference(uri); + entries.add(new Entry(uri, fileReference)); + return fileReference; } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index 4793caad308..e31d90c2800 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -1,6 +1,7 @@ package com.yahoo.searchdefinition; import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; import com.yahoo.path.Path; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.utils.FileSender; @@ -16,7 +17,7 @@ public class DistributableResource { private final String name; private final ByteBuffer blob; private String path; - private String fileReference = ""; + private FileReference fileReference = new FileReference(""); private PathType pathType = PathType.FILE; public PathType getPathType() { @@ -54,18 +55,7 @@ public class DistributableResource { /** Initiate sending of this constant to some services over file distribution */ public void sendTo(Collection<? extends AbstractService> services) { - fileReference = sendToServices(services).value(); - } - private FileReference sendToServices(Collection<? extends AbstractService> services) { - switch (pathType) { - case FILE: - return FileSender.sendFileToServices(path, services); - case URI: - return FileSender.sendUriToServices(path, services); - case BLOB: - return FileSender.sendBlobToServices(blob, services); - } - throw new IllegalArgumentException("Unknown path type " + pathType); + FileSender.send(fileReference, services); } public String getName() { return name; } @@ -73,7 +63,7 @@ public class DistributableResource { public String getFileName() { return path; } public Path getFilePath() { return Path.fromString(path); } public String getUri() { return path; } - public String getFileReference() { return fileReference; } + public String getFileReference() { return fileReference.value(); } public void validate() { switch (pathType) { @@ -88,6 +78,22 @@ public class DistributableResource { } } + void register(FileRegistry fileRegistry) { + switch (pathType) { + case FILE: + fileReference = fileRegistry.addFile(path); + break; + case URI: + fileReference = fileRegistry.addUri(path); + break; + case BLOB: + fileReference = fileRegistry.addBlob(blob); + break; + default: + throw new IllegalArgumentException("Unknown path type " + pathType); + } + } + public String toString() { StringBuilder b = new StringBuilder(); b.append("resource '").append(name).append(" of type '").append(pathType) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java index 518221aac96..9d8e8b90ceb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java @@ -18,6 +18,7 @@ public class LargeRankExpressions { public void add(RankExpressionBody expression) { expression.validate(); + expression.register(fileRegistry); String name = expression.getName(); if (expressions.containsKey(name)) { throw new IllegalArgumentException("Rank expression '" + name + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java index 96ee8c0c2f4..cb61b8c9cec 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java @@ -24,6 +24,7 @@ public class OnnxModels { } public void add(OnnxModel model) { model.validate(); + model.register(fileRegistry); String name = model.getName(); models.put(name, model); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java index a61e1ae3efb..8ccc0ef429e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java @@ -26,6 +26,7 @@ public class RankingConstants { public void add(RankingConstant constant) { constant.validate(); + constant.register(fileRegistry); String name = constant.getName(); if (constants.containsKey(name)) throw new IllegalArgumentException("Ranking constant '" + name + "' defined twice"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index ea0452a6c49..4f8518b0c32 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -8,7 +8,6 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.defaults.Defaults; -import java.nio.ByteBuffer; import java.util.Collection; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -400,7 +399,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon public void setVespaMallocDebugStackTrace(String s) { vespaMallocDebugStackTrace = s; } public String getMMapNoCoreEnvVariable() { - return (getMMapNoCoreLimit() >= 0l) + return (getMMapNoCoreLimit() >= 0L) ? "VESPA_MMAP_NOCORE_LIMIT=" + getMMapNoCoreLimit() + " " : ""; } @@ -457,9 +456,14 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon /** * Add the given file to the application's file distributor. * - * @param relativePath path to the file, relative to the app package - * @return the file reference hash + * @param reference file reference (hash) */ + public void send(FileReference reference) { + Host host = null; + if (getHost() != null) // false when running application tests without hosts + host = getHost().getHost(); + getRoot().getFileDistributor().sendFileReference(reference, host); + } public FileReference sendFile(String relativePath) { Host host = null; if (getHost() != null) // false when running application tests without hosts @@ -467,14 +471,6 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon return getRoot().getFileDistributor().sendFileToHost(relativePath, host); } - public FileReference sendUri(String uri) { - return getRoot().getFileDistributor().sendUriToHost(uri, getHost().getHost()); - } - - public FileReference sendBlob(ByteBuffer blob) { - return getRoot().getFileDistributor().sendBlobToHost(blob, getHost().getHost()); - } - /** The service HTTP port for health status */ public int getHealthPort() { return -1;} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index cf2ff2f5c3b..0f1aaca4454 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -5,7 +5,6 @@ import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.vespa.model.Host; -import java.nio.ByteBuffer; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; @@ -41,24 +40,8 @@ public class FileDistributor { return addFileReference(fileRegistry.addFile(relativePath), host); } - /** - * Adds the given file to the associated application packages' registry of file and marks the file - * for distribution to the given host. - * - * @return the reference to the file, created by the application package - */ - public FileReference sendUriToHost(String uri, Host host) { - return addFileReference(fileRegistry.addUri(uri), host); - } - - /** - * Adds the given blob to the associated application packages' registry of file and marks the file - * for distribution to the given host. - * - * @return the reference to the file, created by the application package - */ - public FileReference sendBlobToHost(ByteBuffer blob, Host host) { - return addFileReference(fileRegistry.addBlob(blob), host); + public void sendFileReference(FileReference reference, Host host) { + addFileReference(reference, host); } private FileReference addFileReference(FileReference reference, Host host) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 52edec7114b..b7bb9f10ade 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.model.AbstractService; import java.io.Serializable; -import java.nio.ByteBuffer; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -28,50 +27,40 @@ public class FileSender implements Serializable { /** * Send the given file to all given services. * - * @param relativePath The path to the file, relative to the app pkg. + * @param fileReference The file reference to send. * @param services The services to send the file to. - * @return The file reference that the file was given, never null. * @throws IllegalStateException if services is empty. */ - public static FileReference sendFileToServices(String relativePath, - Collection<? extends AbstractService> services) { - if (services.isEmpty()) { - throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " + - "using 'count' instead of <node> tags."); - } - - FileReference fileref = null; - for (AbstractService service : services) { - // The same reference will be returned from each call. - fileref = service.sendFile(relativePath); - } - return fileref; - } - - public static FileReference sendUriToServices(String uri, Collection<? extends AbstractService> services) { + public static void send(FileReference fileReference, Collection<? extends AbstractService> services) { if (services.isEmpty()) { throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " + "using 'count' instead of <node> tags."); } - FileReference fileref = null; for (AbstractService service : services) { // The same reference will be returned from each call. - fileref = service.sendUri(uri); + service.send(fileReference); } - return fileref; } - - public static FileReference sendBlobToServices(ByteBuffer blob, Collection<? extends AbstractService> services) { + /** + * Send the given file to all given services. + * + * @param relativePath The path to the file, relative to the app pkg. + * @param services The services to send the file to. + * @return The file reference that the file was given, never null. + * @throws IllegalStateException if services is empty. + */ + public static FileReference sendFileToServices(String relativePath, + Collection<? extends AbstractService> services) { if (services.isEmpty()) { throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " + - "using 'count' instead of <node> tags."); + "using 'count' instead of <node> tags."); } FileReference fileref = null; for (AbstractService service : services) { // The same reference will be returned from each call. - fileref = service.sendBlob(blob); + fileref = service.sendFile(relativePath); } return fileref; } |