diff options
9 files changed, 23 insertions, 29 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java index 358d08ded25..1817f09ae46 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java @@ -35,9 +35,8 @@ public class MockFileRegistry implements FileRegistry { } @Override - public FileReference addBlob(ByteBuffer blob) { - long blobHash = XXHashFactory.fastestJavaInstance().hash64().hash(blob, 0); - String relativePath = Long.toHexString(blobHash) + ".blob"; + public FileReference addBlob(String name, ByteBuffer blob) { + String relativePath = "./" + name; FileReference fileReference = new FileReference(relativePath); entries.add(new Entry(relativePath, fileReference)); return fileReference; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java index a0b1402f559..376bb889dfe 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java @@ -16,7 +16,7 @@ public interface FileRegistry { FileReference addFile(String relativePath); FileReference addUri(String uri); - FileReference addBlob(ByteBuffer blob); + FileReference addBlob(String name, ByteBuffer blob); default FileReference addApplicationPackage() { return addFile(""); } /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index e31d90c2800..7fd36b0d142 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -38,6 +38,7 @@ public class DistributableResource { Objects.requireNonNull(blob, "Blob cannot be null"); this.name = name; this.blob = blob; + path = name; pathType = PathType.BLOB; } @@ -87,7 +88,7 @@ public class DistributableResource { fileReference = fileRegistry.addUri(path); break; case BLOB: - fileReference = fileRegistry.addBlob(blob); + fileReference = fileRegistry.addBlob(path, blob); break; default: throw new IllegalArgumentException("Unknown path type " + pathType); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 07f26f0431d..a64fb2360a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -37,6 +37,7 @@ import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -143,27 +144,22 @@ public class RankSetupValidator extends Validator { writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifcb.build()); } - private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException { - List<String> config = new ArrayList<>(); - - // Assist verify-ranksetup in finding the actual ONNX model files - for (OnnxModel model : db.getDerivedConfiguration().getSearch().onnxModels().asMap().values()) { + private void writeExtraVerifyRanksetupConfig(List<String> config, Collection<? extends DistributableResource> resources) { + for (DistributableResource model : resources) { String modelPath = getFileRepositoryPath(model.getFilePath().getName(), model.getFileReference()); int index = config.size() / 2; config.add(String.format("file[%d].ref \"%s\"", index, model.getFileReference())); config.add(String.format("file[%d].path \"%s\"", index, modelPath)); log.log(Level.INFO, index + ": " + model.getPathType() + " -> " + model.getName() + " -> " + modelPath + " -> " + model.getFileReference()); } + } - for (RankExpressionBody expr : db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()) { - int index = config.size() / 2; - String modelPath = (expr.getPathType() == DistributableResource.PathType.BLOB) - ? getFileRepositoryPath(expr.getName(), expr.getFileReference()) - : getFileRepositoryPath(expr.getFilePath().getName(), expr.getFileReference()); - config.add(String.format("file[%d].ref \"%s\"", index, expr.getFileReference())); - config.add(String.format("file[%d].path \"%s\"", index, modelPath)); - log.log(Level.INFO, index + ": " + expr.getPathType() + " -> " + expr.getName() + " -> " + modelPath + " -> " + expr.getFileReference()); - } + private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException { + List<String> config = new ArrayList<>(); + + // Assist verify-ranksetup in finding the actual ONNX model files + writeExtraVerifyRanksetupConfig(config, db.getDerivedConfiguration().getSearch().onnxModels().asMap().values()); + writeExtraVerifyRanksetupConfig(config, db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()); String configContent = config.isEmpty() ? "" : StringUtilities.implodeMultiline(config); IOUtils.writeFile(dir + "verify-ranksetup.cfg", configContent, false); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java index 2133a9ba899..777217b03fa 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java @@ -53,7 +53,7 @@ public class FileSenderTest { } @Override - public FileReference addBlob(ByteBuffer blob) { + public FileReference addBlob(String name, ByteBuffer blob) { return null; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index 0d0f6d09582..459d8e51853 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -100,8 +100,7 @@ public class FileDBRegistry implements FileRegistry { } @Override - public FileReference addBlob(ByteBuffer blob) { - String blobName = FileRegistry.blobName(blob); + public FileReference addBlob(String blobName, ByteBuffer blob) { String relativePath = blobToRelativeFile(blobName); synchronized (this) { Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(blobName)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java index 10ac54dc757..2e5da471da0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java @@ -25,7 +25,7 @@ public class FileDBRegistryTestCase { private static final String APP = "src/test/apps/zkapp"; private static final String FOO_FILE = "files/foo.json"; private static final String NO_FOO_FILE = "files/no_foo.json"; - private static final String BLOB_NAME = "c5674b55c15c9c95"; + private static final String BLOB_NAME = "myblob.name"; private static final FileReference BLOB_REF = new FileReference("b3424b78130fc005"); private static final FileReference FOO_REF = new FileReference("b5ce94ca1feae86c"); @Test @@ -41,7 +41,7 @@ public class FileDBRegistryTestCase { } catch (IllegalArgumentException e) { assertEquals("src/test/apps/zkapp/files/no_foo.json (No such file or directory)", e.getCause().getMessage()); } - assertEquals(BLOB_REF, fileRegistry.addBlob(ByteBuffer.wrap(BLOB.getBytes(StandardCharsets.UTF_8)))); + assertEquals(BLOB_REF, fileRegistry.addBlob(BLOB_NAME, ByteBuffer.wrap(BLOB.getBytes(StandardCharsets.UTF_8)))); String serializedRegistry = FileDBRegistry.exportRegistry(fileRegistry); FileDBRegistry importedRegistry = FileDBRegistry.create(fileManager, new StringReader(serializedRegistry)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java index 8b3227e3a69..939838cba10 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java @@ -50,9 +50,8 @@ public class MockFileRegistry implements FileRegistry { } @Override - public FileReference addBlob(ByteBuffer blob) { - long blobHash = XXHashFactory.fastestJavaInstance().hash64().hash(blob, 0); - String relativePath = "./" + Long.toHexString(blobHash) + ".blob"; + public FileReference addBlob(String name, ByteBuffer blob) { + String relativePath = "./" + name; FileReference fileReference = addFileInterface.addBlob(blob, relativePath); entries.add(new Entry(relativePath, fileReference)); diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java index 977443c8e5b..df31e454cd7 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java @@ -79,8 +79,8 @@ public class LocalFileDb implements FileAcquirer, FileRegistry { } @Override - public FileReference addBlob(ByteBuffer blob) { - throw new RuntimeException("addBlob(ByteBuffer blob) is not implemented here."); + public FileReference addBlob(String name, ByteBuffer blob) { + throw new RuntimeException("addBlob(String name, ByteBuffer blob) is not implemented here."); } private static Constructor<FileReference> createFileReferenceConstructor() { |