summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@gmail.com>2022-09-01 12:59:06 +0200
committerJon Bratseth <bratseth@gmail.com>2022-09-01 12:59:06 +0200
commit86a39ab21db4a92b46944709251ccab58ef39370 (patch)
tree5cac1fc0065019c5161b4000ba28f4382322a664
parent0652206123f2a65156bf21663aac0e75b4f43b73 (diff)
Use Path as user type for models
-rw-r--r--config-lib/abi-spec.json7
-rw-r--r--config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java8
-rw-r--r--config-lib/src/main/java/com/yahoo/config/ModelNode.java14
-rw-r--r--config-lib/src/main/java/com/yahoo/config/ModelReference.java16
-rw-r--r--config-lib/src/main/java/com/yahoo/config/PathNode.java2
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java2
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java2
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java2
-rw-r--r--configgen/src/test/resources/allfeatures.reference15
-rw-r--r--model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java4
10 files changed, 36 insertions, 36 deletions
diff --git a/config-lib/abi-spec.json b/config-lib/abi-spec.json
index 19dfd8535ae..a8bdc756f08 100644
--- a/config-lib/abi-spec.json
+++ b/config-lib/abi-spec.json
@@ -319,14 +319,12 @@
"methods": [
"public void <init>()",
"public void <init>(com.yahoo.config.ModelReference)",
- "public com.yahoo.config.ModelReference value()",
"public java.lang.String getValue()",
"public java.lang.String toString()",
"protected boolean doSetValue(java.lang.String)",
"public com.yahoo.config.ModelReference getModelReference()",
"public static java.util.List toModelReferences(java.util.List)",
- "public static java.util.Map toModelReferenceMap(java.util.Map)",
- "public bridge synthetic java.lang.Object value()"
+ "public static java.util.Map toModelReferenceMap(java.util.Map)"
],
"fields": []
},
@@ -344,13 +342,14 @@
"public com.yahoo.config.ModelReference withModelId(java.util.Optional)",
"public com.yahoo.config.ModelReference withUrl(java.util.Optional)",
"public com.yahoo.config.ModelReference withPath(java.util.Optional)",
- "public java.lang.String value()",
+ "public java.nio.file.Path value()",
"public boolean equals(java.lang.Object)",
"public int hashCode()",
"public java.lang.String toString()",
"public static com.yahoo.config.ModelReference fromModelId(java.lang.String)",
"public static com.yahoo.config.ModelReference fromUrl(java.lang.String)",
"public static com.yahoo.config.ModelReference fromPath(java.lang.String)",
+ "public static com.yahoo.config.ModelReference valueOf(java.nio.file.Path)",
"public static com.yahoo.config.ModelReference valueOf(java.lang.String)"
],
"fields": []
diff --git a/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java b/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java
index bdabc045e18..ddbed4258dc 100644
--- a/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java
+++ b/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java
@@ -78,11 +78,11 @@ public class LeafNodeVector<REAL, NODE extends LeafNode<REAL>> extends NodeVecto
return new LeafNodeVector<>(files, new UrlNode());
}
- public static LeafNodeVector<ModelReference, ModelNode> createModelNodeVector(Collection<ModelReference> values) {
- List<ModelReference> models = new ArrayList<>();
+ public static LeafNodeVector<Path, ModelNode> createModelNodeVector(Collection<ModelReference> values) {
+ List<Path> modelPaths = new ArrayList<>();
for (ModelReference modelReference : values)
- models.add(modelReference);
- return new LeafNodeVector<>(models, new ModelNode());
+ modelPaths.add(modelReference.value());
+ return new LeafNodeVector<>(modelPaths, new ModelNode());
}
}
diff --git a/config-lib/src/main/java/com/yahoo/config/ModelNode.java b/config-lib/src/main/java/com/yahoo/config/ModelNode.java
index 775c4a12e82..2748ef8c7e9 100644
--- a/config-lib/src/main/java/com/yahoo/config/ModelNode.java
+++ b/config-lib/src/main/java/com/yahoo/config/ModelNode.java
@@ -12,19 +12,19 @@ import java.util.Map;
*
* @author bratseth
*/
-public class ModelNode extends LeafNode<ModelReference> {
+public class ModelNode extends LeafNode<Path> {
+
+ private final ModelReference reference;
public ModelNode() {
this.value = null;
+ this.reference = null;
}
public ModelNode(ModelReference modelReference) {
super(true);
- this.value = modelReference;
- }
-
- public ModelReference value() {
- return value;
+ this.value = modelReference.value();
+ this.reference = modelReference;
}
@Override
@@ -43,7 +43,7 @@ public class ModelNode extends LeafNode<ModelReference> {
}
public ModelReference getModelReference() {
- return value;
+ return reference;
}
public static List<ModelReference> toModelReferences(List<ModelNode> modelNodes) {
diff --git a/config-lib/src/main/java/com/yahoo/config/ModelReference.java b/config-lib/src/main/java/com/yahoo/config/ModelReference.java
index c0b610e4df3..7f7082384af 100644
--- a/config-lib/src/main/java/com/yahoo/config/ModelReference.java
+++ b/config-lib/src/main/java/com/yahoo/config/ModelReference.java
@@ -2,6 +2,7 @@
package com.yahoo.config;
import java.io.File;
+import java.nio.file.Path;
import java.util.Objects;
import java.util.Optional;
@@ -45,13 +46,13 @@ public class ModelReference {
return new ModelReference(modelId, url, path);
}
- /** Returns the path to the file containing this model. */
- public String value() {
+ /** Returns the path to the file containing this model, or null if not available. */
+ public Path value() {
if (url.isPresent() && new File(url.get().value()).exists())
- return new File(url.get().value()).getAbsolutePath();
+ return Path.of(url.get().value());
if (path.isPresent())
- return path.get().value();
- throw new IllegalStateException("No url or path is available");
+ return Path.of(path.get().value());
+ return null;
}
@Override
@@ -91,6 +92,11 @@ public class ModelReference {
return new ModelReference(Optional.empty(), Optional.empty(), Optional.of(new FileReference(path)));
}
+ /** Creates a model reference having a path only. */
+ public static ModelReference valueOf(Path path) {
+ return fromPath(path.toString());
+ }
+
/**
* Creates a model reference from a three-part string on the form
* <code>modelId url path</code>
diff --git a/config-lib/src/main/java/com/yahoo/config/PathNode.java b/config-lib/src/main/java/com/yahoo/config/PathNode.java
index b757e569ffe..10100d374d6 100644
--- a/config-lib/src/main/java/com/yahoo/config/PathNode.java
+++ b/config-lib/src/main/java/com/yahoo/config/PathNode.java
@@ -23,8 +23,6 @@ public class PathNode extends LeafNode<Path> {
public PathNode(FileReference fileReference) {
super(true);
this.value = Path.of(fileReference.value());
- if (value.normalize().toString().startsWith(".."))
- throw new IllegalArgumentException("path may not start with '..', but got: " + value);
this.fileReference = fileReference;
}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java
index e7b6455f3e0..78ef17f613a 100644
--- a/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java
+++ b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java
@@ -271,7 +271,7 @@ public class BuilderGenerator {
String bType = builderType(n);
String stringSetter = "";
- if ( ! "String".equals(bType) && ! "FileReference".equals(bType)) {
+ if ( ! "String".equals(bType) && ! "FileReference".equals(bType) && ! "ModelReference".equals(bType)) {
String type = boxedDataType(n);
if ("UrlReference".equals(bType))
type = bType;
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java b/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java
index fd3aa388ed6..5ffe18b1699 100644
--- a/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java
+++ b/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java
@@ -434,7 +434,7 @@ public class ConfigGenerator {
} else if (node instanceof UrlLeaf) {
return "File";
} else if (node instanceof ModelLeaf) {
- return "ModelReference";
+ return "Path";
} else if (node instanceof IntegerLeaf) {
return "int";
} else if (node instanceof LongLeaf) {
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java
index 4141a3b6e09..78977f43bdd 100644
--- a/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java
+++ b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java
@@ -138,7 +138,7 @@ public class JavaClassBuilder implements ClassBuilder {
for (int i = 1;; i++) {
String candidate = (i < basis.length()) ? basis.substring(0, i)
: ReservedWords.INTERNAL_PREFIX + basis + rng.nextInt(Integer.MAX_VALUE);
- if (usedSymbols.contains(candidate) == false) {
+ if ( ! usedSymbols.contains(candidate)) {
return candidate;
}
}
diff --git a/configgen/src/test/resources/allfeatures.reference b/configgen/src/test/resources/allfeatures.reference
index 0961d3aa922..8a681048f65 100644
--- a/configgen/src/test/resources/allfeatures.reference
+++ b/configgen/src/test/resources/allfeatures.reference
@@ -430,9 +430,6 @@ public final class AllfeaturesConfig extends ConfigInstance {
return this;
}
- private Builder modelVal(String __value) {
- return modelVal(ModelReference.valueOf(__value));
- }
public Builder boolarr(Boolean __value) {
boolarr.add(__value);
@@ -774,7 +771,7 @@ public final class AllfeaturesConfig extends ConfigInstance {
private final LeafNodeVector<FileReference, FileNode> filearr;
private final LeafNodeVector<Path, PathNode> pathArr;
private final LeafNodeVector<File, UrlNode> urlArr;
- private final LeafNodeVector<ModelReference, ModelNode> modelArr;
+ private final LeafNodeVector<Path, ModelNode> modelArr;
private final Map<String, IntegerNode> intMap;
private final Map<String, FileNode> pathMap;
private final Map<String, UrlNode> urlMap;
@@ -972,7 +969,7 @@ public final class AllfeaturesConfig extends ConfigInstance {
/**
* @return allfeatures.modelVal
*/
- public ModelReference modelVal() {
+ public Path modelVal() {
return modelVal.value();
}
@@ -1129,7 +1126,7 @@ public final class AllfeaturesConfig extends ConfigInstance {
/**
* @return allfeatures.modelArr[]
*/
- public List<ModelReference> modelArr() {
+ public List<Path> modelArr() {
return modelArr.asList();
}
@@ -1137,7 +1134,7 @@ public final class AllfeaturesConfig extends ConfigInstance {
* @param i the index of the value to return
* @return allfeatures.modelArr[]
*/
- public ModelReference modelArr(int i) {
+ public Path modelArr(int i) {
return modelArr.get(i).value();
}
@@ -1189,7 +1186,7 @@ public final class AllfeaturesConfig extends ConfigInstance {
/**
* @return allfeatures.modelMap{}
*/
- public Map<String, ModelReference> modelMap() {
+ public Map<String, Path> modelMap() {
return LeafNodeMaps.asValueMap(modelMap);
}
@@ -1197,7 +1194,7 @@ public final class AllfeaturesConfig extends ConfigInstance {
* @param key the key of the value to return
* @return allfeatures.modelMap{}
*/
- public ModelReference modelMap(String key) {
+ public Path modelMap(String key) {
return modelMap.get(key).value();
}
diff --git a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java
index 3dd8d7eefc4..149598ee2dd 100644
--- a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java
+++ b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java
@@ -57,8 +57,8 @@ public class BertBaseEmbedder implements Embedder {
options.setInterOpThreads(modifyThreadCount(config.onnxInterOpThreads()));
options.setIntraOpThreads(modifyThreadCount(config.onnxIntraOpThreads()));
- tokenizer = new WordPieceEmbedder.Builder(config.tokenizerVocab().value()).build();
- evaluator = new OnnxEvaluator(config.transformerModel().value(), options);
+ tokenizer = new WordPieceEmbedder.Builder(config.tokenizerVocab().toString()).build();
+ evaluator = new OnnxEvaluator(config.transformerModel().toString(), options);
validateModel();
}