diff options
author | Harald Musum <musum@verizonmedia.com> | 2021-11-05 19:56:45 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-11-05 19:56:45 +0100 |
commit | 89d3340928b4d1014aebb3cc8919614fe3b3cf25 (patch) | |
tree | b742bfb8ddf589618d459e675f6449b3e440af9d | |
parent | b4d75eb34c424d1f5e2b9b57a43be51a9c8e19b9 (diff) | |
parent | 63a6651c9d3e74218f3eec1458b7c7f46de0471d (diff) |
Merge pull request #19889 from vespa-engine/balder/step-one-in-making-final-and-simpler
- Move blob to LargeRankExpressions and prepare som simplifying Distr…
4 files changed, 60 insertions, 54 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index 08194c578e7..77133d9b03d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -13,43 +13,44 @@ import java.util.Objects; public class DistributableResource { - public enum PathType { FILE, URI, BLOB }; + public enum PathType { FILE, URI, BLOB } /** The search definition-unique name of this constant */ private final String name; - private final ByteBuffer blob; + //TODO Make path/pathType final + private PathType pathType; private String path; private FileReference fileReference = new FileReference(""); - private PathType pathType = PathType.FILE; public PathType getPathType() { return pathType; } public DistributableResource(String name) { - this.name = name; - blob = null; + this(name, null, PathType.FILE); } public DistributableResource(String name, String path) { + this(name, path, PathType.FILE); + } + public DistributableResource(String name, String path, PathType type) { this.name = name; this.path = path; - blob = null; + this.pathType = type; } public DistributableResource(String name, ByteBuffer blob) { - Objects.requireNonNull(name, "Blob name cannot be null"); - Objects.requireNonNull(blob, "Blob cannot be null"); this.name = name; - this.blob = blob; path = name + ".lz4"; pathType = PathType.BLOB; } + //TODO Remove and make path/pathType final public void setFileName(String fileName) { Objects.requireNonNull(fileName, "Filename cannot be null"); this.path = fileName; this.pathType = PathType.FILE; } + //TODO Remove and make path/pathType final public void setUri(String uri) { Objects.requireNonNull(uri, "uri cannot be null"); this.path = uri; @@ -62,7 +63,6 @@ public class DistributableResource { } public String getName() { return name; } - public ByteBuffer getBlob() { return blob; } public String getFileName() { return path; } public Path getFilePath() { return Path.fromString(path); } public String getUri() { return path; } @@ -75,9 +75,6 @@ public class DistributableResource { if (path == null || path.isEmpty()) throw new IllegalArgumentException("Distributable URI/FILE resource must have a file or uri."); break; - case BLOB: - if (blob == null) - throw new IllegalArgumentException("Distributable BLOB can not be null."); } } @@ -89,14 +86,15 @@ public class DistributableResource { case URI: fileReference = fileRegistry.addUri(path); break; - case BLOB: - fileReference = fileRegistry.addBlob(path, blob); - break; default: throw new IllegalArgumentException("Unknown path type " + pathType); } } + protected void register(FileRegistry fileRegistry, ByteBuffer blob) { + fileReference = fileRegistry.addBlob(path, blob); + } + @Override public String toString() { return "resource '" + name + " of type '" + pathType + "' with ref '" + fileReference + "'"; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java index b3dec554cdc..815e4bbf359 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java @@ -1,11 +1,29 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.application.api.FileRegistry; + import java.nio.ByteBuffer; +import java.util.Objects; public class RankExpressionBody extends DistributableResource { - public RankExpressionBody(String name, ByteBuffer body) { - super(name, body); + private final ByteBuffer blob; + + public RankExpressionBody(String name, ByteBuffer blob) { + super(name, blob); + Objects.requireNonNull(blob, "Blob cannot be null"); + this.blob = blob; + } + public ByteBuffer getBlob() { return blob; } + public void validate() { + // Remove once pathType is final + if (getPathType() != PathType.BLOB) { + throw new IllegalArgumentException("PathType must be BLOB."); + } + } + + void register(FileRegistry fileRegistry) { + register(fileRegistry, blob); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java index 7f29034157b..377e19eedf2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java @@ -19,7 +19,10 @@ public class RankingConstant extends DistributableResource { } public RankingConstant(String name, TensorType type, String fileName) { - super(name, fileName); + this(name, type, fileName, PathType.FILE); + } + public RankingConstant(String name, TensorType type, String fileName, PathType pathType) { + super(name, fileName, pathType); this.tensorType = type; validate(); } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index fe522494baf..5bcc033fa37 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -29,6 +29,7 @@ import com.yahoo.documentmodel.*; import com.yahoo.compress.Compressor; import com.yahoo.compress.CompressionType; import com.yahoo.searchdefinition.Application; +import com.yahoo.searchdefinition.DistributableResource; import com.yahoo.searchdefinition.document.*; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; @@ -1945,8 +1946,8 @@ Object onnxModelItem(OnnxModel onnxModel) : } { ( - (<FILE> <COLON> path = filePath() { } (<NL>)*) { onnxModel.setFileName(path); } | - (<URI> <COLON> path = uriPath() { } (<NL>)*) { onnxModel.setUri(path); } | + (path = fileItem()) { onnxModel.setFileName(path); } | + (path = uriItem()) { onnxModel.setUri(path); } | (<ONNX_INPUT_SL>) { String name = token.image.substring(5, token.image.lastIndexOf(":")).trim(); if (name.startsWith("\"")) { name = name.substring(1, name.length() - 1); } @@ -1973,56 +1974,42 @@ Object onnxModelItem(OnnxModel onnxModel) : void rankingConstant(Schema schema) : { String name; - RankingConstant constant; + String path = null; + DistributableResource.PathType pathType = DistributableResource.PathType.FILE; + TensorType type = null; } { - ( <CONSTANT> name = identifier() - { - constant = new RankingConstant(name); - } - lbrace() (rankingConstantItem(constant) (<NL>)*)+ <RBRACE> ) + ( <CONSTANT> name = identifier() lbrace() + (path = fileItem() { pathType = DistributableResource.PathType.FILE; } + | path = uriItem() { pathType = DistributableResource.PathType.URI; } + | type = tensorTypeWithPrefix(rankingConstantErrorMessage(name)) (<NL>)* + )+ + <RBRACE> + ) { if (documentsOnly) return; - schema.rankingConstants().add(constant); + schema.rankingConstants().add(new RankingConstant(name, type, path, pathType)); } } -/** - * This rule consumes a constant block. - * - * @param constant The constant to modify. - * @return Null. - */ -Object rankingConstantItem(RankingConstant constant) : +String fileItem() : { - String path = null; - TensorType type = null; + String path; } { - ( (<FILE> <COLON> path = filePath() { } (<NL>)*) { constant.setFileName(path); } - | (<URI> <COLON> path = uriPath() { } (<NL>)*) { constant.setUri(path); } - | type = tensorTypeWithPrefix(rankingConstantErrorMessage(constant.getName())) (<NL>)* { constant.setType(type); } - ) - { - return null; - } + (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = token.image; } { } (<NL>)*) { return path; } } - -String rankingConstantErrorMessage(String name) : {} +String uriItem() : { - { return "For ranking constant ' " + name + "'"; } + String path; } - -String filePath() : { } { - ( <FILE_PATH> | <STRING> | <IDENTIFIER>) - { return token.image; } + (<URI> <COLON> ( <URI_PATH> ) { path = token.image; } (<NL>)*) { return path; } } -String uriPath() : { } +String rankingConstantErrorMessage(String name) : {} { - ( <URI_PATH> ) - { return token.image; } + { return "For ranking constant ' " + name + "'"; } } /** |