diff options
13 files changed, 66 insertions, 11 deletions
diff --git a/config-lib/abi-spec.json b/config-lib/abi-spec.json index 8a1cfa9834e..dd4cf995f64 100644 --- a/config-lib/abi-spec.json +++ b/config-lib/abi-spec.json @@ -468,6 +468,7 @@ "methods": [ "public void <init>()", "public void <init>(com.yahoo.config.UrlReference)", + "public void <init>(java.lang.String)", "public java.io.File value()", "public java.lang.String toString()", "public java.lang.String getValue()", @@ -489,6 +490,7 @@ "methods": [ "public void <init>(java.lang.String)", "public java.lang.String value()", + "public static com.yahoo.config.UrlReference valueOf(java.lang.String)", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" diff --git a/config-lib/src/main/java/com/yahoo/config/UrlNode.java b/config-lib/src/main/java/com/yahoo/config/UrlNode.java index d71439ebb9a..ace94f561ff 100644 --- a/config-lib/src/main/java/com/yahoo/config/UrlNode.java +++ b/config-lib/src/main/java/com/yahoo/config/UrlNode.java @@ -28,6 +28,10 @@ public class UrlNode extends LeafNode<File> { this.value = new File(url.value()); } + public UrlNode(String url) { + this(new UrlReference(url)); + } + public File value() { return value; } @@ -60,4 +64,14 @@ public class UrlNode extends LeafNode<File> { Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getUrlReference())); } + @Override + void serialize(String name, Serializer serializer) { + serializer.serialize(name, url.value()); + } + + @Override + void serialize(Serializer serializer) { + serializer.serialize(url.value()); + } + } diff --git a/config-lib/src/main/java/com/yahoo/config/UrlReference.java b/config-lib/src/main/java/com/yahoo/config/UrlReference.java index 470828c7d0c..1adab8aaa53 100755 --- a/config-lib/src/main/java/com/yahoo/config/UrlReference.java +++ b/config-lib/src/main/java/com/yahoo/config/UrlReference.java @@ -21,6 +21,10 @@ public final class UrlReference { return value; } + public static UrlReference valueOf(String value) { + return new UrlReference(value); + } + @Override public int hashCode() { return value.hashCode(); diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java index 46b710cdcf9..edf4c0270a7 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java @@ -9,6 +9,7 @@ import com.yahoo.foo.IntConfig; import com.yahoo.foo.MaptypesConfig; import com.yahoo.foo.SimpletypesConfig; import com.yahoo.foo.StructtypesConfig; +import com.yahoo.foo.UrlConfig; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.text.StringUtilities; @@ -138,6 +139,12 @@ public class ConfigPayloadTest { assertThat(payload.toString(true), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } + @Test + public void test_serialize_url_fields() { + ConfigPayload payload = ConfigPayload.fromInstance(new UrlConfig(new UrlConfig.Builder())); + assertThat(payload.toString(true), is("{\"urlVal\":\"http://vespa.ai\"}")); + } + @Test(expected=RuntimeException.class) public void test_double_leaf_illegal_string() { createSimpletypesConfig("doubleval", "illegal"); diff --git a/config/src/test/resources/configs/def-files/url.def b/config/src/test/resources/configs/def-files/url.def new file mode 100755 index 00000000000..f18a99efa51 --- /dev/null +++ b/config/src/test/resources/configs/def-files/url.def @@ -0,0 +1,4 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=foo + +urlVal url default=http://vespa.ai diff --git a/configdefinitions/src/main/java/com/yahoo/embedding/package-info.java b/configdefinitions/src/main/java/com/yahoo/embedding/package-info.java new file mode 100644 index 00000000000..1842b80aba9 --- /dev/null +++ b/configdefinitions/src/main/java/com/yahoo/embedding/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.embedding; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 3735a8ae57a..33ceb5338d3 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -74,3 +74,4 @@ vespa_generate_config(configdefinitions all-clusters-bucket-spaces.def) install_config_definition(all-clusters-bucket-spaces.def vespa.config.content.all-clusters-bucket-spaces.def) vespa_generate_config(configdefinitions stateserver.def) install_config_definition(stateserver.def vespa.config.core.stateserver.def) +install_config_definition(embedding.bert-base-embedder.def embedding.bert-base-embedder.def) diff --git a/model-integration/src/main/resources/configdefinitions/bert-base-embedder.def b/configdefinitions/src/vespa/embedding.bert-base-embedder.def index 7e3ff151466..a37599de411 100644 --- a/model-integration/src/main/resources/configdefinitions/bert-base-embedder.def +++ b/configdefinitions/src/vespa/embedding.bert-base-embedder.def @@ -1,7 +1,8 @@ -package=ai.vespa.embedding + +namespace=embedding # Transformer model settings -transformerModelUrl url +transformerModelUrl url default=https://data.vespa.oath.cloud/onnx_models/sentence_all_MiniLM_L6_v2.onnx # Max length of token sequence model can handle transformerMaxTokens int default=384 @@ -23,5 +24,4 @@ onnxInterOpThreads int default=1 onnxIntraOpThreads int default=-4 # n=number of threads -> n<0: CPUs/(-n), n==0: CPUs, n>0: n # Settings for wordpiece tokenizer -tokenizerVocabUrl url - +tokenizerVocabUrl url default=https://data.vespa.oath.cloud/onnx_models/bert-base-uncased-vocab.txt diff --git a/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java index 937d04b35bd..a0717a1060f 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java @@ -204,7 +204,7 @@ public class BuilderGenerator { } private static String privateLeafNodeSetter(LeafCNode n) { - if ("String".equals(builderType(n)) || "FileReference".equals(builderType(n)) || "UrlReference".equals(builderType(n))) { + if ("String".equals(builderType(n)) || "FileReference".equals(builderType(n))) { return ""; } else { return "\n\n" + // @@ -247,7 +247,7 @@ public class BuilderGenerator { } private static String privateLeafMapSetter(CNode n) { - if ("String".equals(builderType(n)) || "FileReference".equals(builderType(n)) || "UrlReference".equals(builderType(n))) { + if ("String".equals(builderType(n)) || "FileReference".equals(builderType(n))) { return ""; } else { return "\n\n" + // @@ -269,10 +269,15 @@ public class BuilderGenerator { : ""; String bType = builderType(n); - String stringSetter = "String".equals(bType) || "FileReference".equals(bType) || "UrlReference".equals(bType) ? "" - : String.format("\nprivate Builder %s(String %svalue) {\n" + // - " return %s(%s.valueOf(%svalue));\n" + // - "}", name, INTERNAL_PREFIX, name, boxedDataType(n), INTERNAL_PREFIX); + String stringSetter = ""; + if ( ! "String".equals(bType) && ! "FileReference".equals(bType)) { + String type = boxedDataType(n); + if ("UrlReference".equals(bType)) + type = bType; + stringSetter = String.format("\nprivate Builder %s(String %svalue) {\n" + // + " return %s(%s.valueOf(%svalue));\n" + // + "}", name, INTERNAL_PREFIX, name, type, INTERNAL_PREFIX); + } String getNullGuard = bType.equals(boxedBuilderType(n)) ? String.format( "\nif (%svalue == null) throw new IllegalArgumentException(\"Null value is not allowed.\");", INTERNAL_PREFIX) : ""; diff --git a/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java b/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java index d0a535ff2fc..a50bb758be7 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java @@ -218,7 +218,7 @@ public abstract class LeafCNode extends CNode { } } - public static class UrlLeaf extends NoClassNoDefaultLeafCNode { + public static class UrlLeaf extends NoClassLeafCNode { UrlLeaf(InnerCNode parent, String name) { super(parent, name); } diff --git a/configgen/src/test/resources/allfeatures.reference b/configgen/src/test/resources/allfeatures.reference index 7698ea6727c..21f49d21413 100644 --- a/configgen/src/test/resources/allfeatures.reference +++ b/configgen/src/test/resources/allfeatures.reference @@ -406,6 +406,9 @@ public final class AllfeaturesConfig extends ConfigInstance { return this; } + private Builder urlVal(String __value) { + return urlVal(UrlReference.valueOf(__value)); + } public Builder boolarr(Boolean __value) { boolarr.add(__value); @@ -527,6 +530,10 @@ public final class AllfeaturesConfig extends ConfigInstance { return this; } + private Builder urlArr(String __value) { + return urlArr(UrlReference.valueOf(__value)); + } + public Builder intMap(String __key, Integer __value) { intMap.put(__key, __value); return this; @@ -561,6 +568,10 @@ public final class AllfeaturesConfig extends ConfigInstance { return this; } + private Builder urlMap(String __key, String __value) { + return urlMap(__key, UrlReference.valueOf(__value)); + } + public Builder basic_struct(Basic_struct.Builder __builder) { basic_struct = __builder; return this; diff --git a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java index 42e3d653359..1831903d626 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java @@ -2,6 +2,7 @@ package ai.vespa.embedding; import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.modelintegration.evaluator.OnnxEvaluatorOptions; +import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.component.annotation.Inject; import com.yahoo.language.process.Embedder; import com.yahoo.language.wordpiece.WordPieceEmbedder; diff --git a/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java b/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java index 0ecc78f7668..464e5941e89 100644 --- a/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java +++ b/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java @@ -2,6 +2,7 @@ package ai.vespa.embedding; import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import com.yahoo.config.UrlReference; +import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import org.junit.Test; |