diff options
author | Jon Bratseth <bratseth@gmail.com> | 2021-09-25 14:50:33 +0000 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2021-09-25 14:50:33 +0000 |
commit | 1abb5adacbdbcfad7070243630164e4d31f68773 (patch) | |
tree | 069d42abf56db3a14bcff1f01df6291728daa32c | |
parent | 5ff3a5a4cbbd43180d821700f572eae73720fc17 (diff) |
Separate component from linguistics
-rw-r--r-- | CMakeLists.txt | 1 | ||||
-rw-r--r-- | container-disc/pom.xml | 1 | ||||
-rw-r--r-- | dist/vespa.spec | 1 | ||||
-rw-r--r-- | linguistics-components/.gitignore | 5 | ||||
-rw-r--r-- | linguistics-components/CMakeLists.txt | 5 | ||||
-rw-r--r-- | linguistics-components/OWNERS | 2 | ||||
-rw-r--r-- | linguistics-components/README | 4 | ||||
-rw-r--r-- | linguistics-components/abi-spec.json | 189 | ||||
-rw-r--r-- | linguistics-components/pom.xml | 80 | ||||
-rw-r--r-- | linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Model.java (renamed from linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java) | 0 | ||||
-rw-r--r-- | linguistics-components/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java (renamed from linguistics/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java) | 0 | ||||
-rw-r--r-- | linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Scoring.java (renamed from linguistics/src/main/java/com/yahoo/language/sentencepiece/Scoring.java) | 0 | ||||
-rw-r--r-- | linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java (renamed from linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java) | 0 | ||||
-rw-r--r-- | linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java (renamed from linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java) | 0 | ||||
-rw-r--r-- | linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java (renamed from linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java) | 0 | ||||
-rw-r--r-- | linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Trie.java (renamed from linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java) | 0 | ||||
-rw-r--r-- | linguistics-components/src/main/java/com/yahoo/language/sentencepiece/package-info.java (renamed from linguistics/src/main/java/com/yahoo/language/sentencepiece/package-info.java) | 2 | ||||
-rw-r--r-- | linguistics-components/src/main/protobuf/sentencepiece_model.proto (renamed from linguistics/src/main/protobuf/sentencepiece_model.proto) | 0 | ||||
-rw-r--r-- | linguistics-components/src/main/resources/configdefinitions/sentence-piece.def (renamed from linguistics/src/main/resources/configdefinitions/sentence-piece.def) | 0 | ||||
-rw-r--r-- | linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java (renamed from linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java) | 0 | ||||
-rw-r--r-- | linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java (renamed from linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java) | 0 | ||||
-rw-r--r-- | linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java (renamed from linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java) | 0 | ||||
-rw-r--r-- | linguistics-components/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model (renamed from linguistics/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model) | bin | 400869 -> 400869 bytes | |||
-rw-r--r-- | linguistics-components/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model (renamed from linguistics/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model) | bin | 300865 -> 300865 bytes | |||
-rw-r--r-- | linguistics/abi-spec.json | 187 | ||||
-rw-r--r-- | linguistics/pom.xml | 4 | ||||
-rw-r--r-- | pom.xml | 1 | ||||
-rw-r--r-- | standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java | 2 |
28 files changed, 291 insertions, 193 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 3598d259144..7bae27287f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,6 +96,7 @@ add_subdirectory(jdisc_jetty) add_subdirectory(jrt_test) add_subdirectory(juniper) add_subdirectory(linguistics) +add_subdirectory(linguistics-components) add_subdirectory(logd) add_subdirectory(logserver) add_subdirectory(logforwarder) diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 5debf6c9c02..f72348707c2 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -175,6 +175,7 @@ docprocs-jar-with-dependencies.jar, hosted-zone-api-jar-with-dependencies.jar, jdisc-security-filters-jar-with-dependencies.jar, + linguistics-components-jar-with-dependencies.jar, vespaclient-container-plugin-jar-with-dependencies.jar, vespa-athenz-jar-with-dependencies.jar, security-utils-jar-with-dependencies.jar, diff --git a/dist/vespa.spec b/dist/vespa.spec index aa3d64401d3..0c5267f2847 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -811,6 +811,7 @@ fi %{_prefix}/lib/jars/jdisc_core-jar-with-dependencies.jar %{_prefix}/lib/jars/jdisc-security-filters-jar-with-dependencies.jar %{_prefix}/lib/jars/jersey-*.jar +%{_prefix}/lib/jars/linguistics-components-jar-with-dependencies.jar %{_prefix}/lib/jars/alpn-*.jar %{_prefix}/lib/jars/http2-*.jar %{_prefix}/lib/jars/jetty-*.jar diff --git a/linguistics-components/.gitignore b/linguistics-components/.gitignore new file mode 100644 index 00000000000..8b990078588 --- /dev/null +++ b/linguistics-components/.gitignore @@ -0,0 +1,5 @@ +target +*.iml +*.ipr +*.iws +/pom.xml.build diff --git a/linguistics-components/CMakeLists.txt b/linguistics-components/CMakeLists.txt new file mode 100644 index 00000000000..b53c8001959 --- /dev/null +++ b/linguistics-components/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +install_fat_java_artifact(linguistics-components) + +install_config_definitions() diff --git a/linguistics-components/OWNERS b/linguistics-components/OWNERS new file mode 100644 index 00000000000..cd50f7a263a --- /dev/null +++ b/linguistics-components/OWNERS @@ -0,0 +1,2 @@ +bratseth +arnej27959 diff --git a/linguistics-components/README b/linguistics-components/README new file mode 100644 index 00000000000..e26a51e2f53 --- /dev/null +++ b/linguistics-components/README @@ -0,0 +1,4 @@ +Java library for linguistic operations in Vespa. + +This API is pluggable - multiple implementations may be supplied. +This module contains a default pure Java implementation, "simple".
\ No newline at end of file diff --git a/linguistics-components/abi-spec.json b/linguistics-components/abi-spec.json new file mode 100644 index 00000000000..5b6729c58ef --- /dev/null +++ b/linguistics-components/abi-spec.json @@ -0,0 +1,189 @@ +{ + "com.yahoo.language.sentencepiece.Scoring": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.language.sentencepiece.Scoring[] values()", + "public static com.yahoo.language.sentencepiece.Scoring valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.language.sentencepiece.Scoring highestScore", + "public static final enum com.yahoo.language.sentencepiece.Scoring fewestSegments" + ] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder collapseUnknowns(boolean)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder scoring(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(java.util.List)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig build()" + ], + "fields": [ + "public java.util.List model" + ] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder language(java.lang.String)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder path(com.yahoo.config.FileReference)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model build()" + ], + "fields": [] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Model": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)", + "public java.lang.String language()", + "public java.nio.file.Path path()" + ], + "fields": [] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Producer": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void getConfig(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)" + ], + "fields": [] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum[] values()", + "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore", + "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments" + ] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring": { + "superClass": "com.yahoo.config.EnumNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)" + ], + "fields": [ + "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore", + "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments" + ] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig": { + "superClass": "com.yahoo.config.ConfigInstance", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public static java.lang.String getDefVersion()", + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)", + "public boolean collapseUnknowns()", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum scoring()", + "public java.util.List model()", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model model(int)" + ], + "fields": [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String CONFIG_DEF_VERSION", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, + "com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void addModel(com.yahoo.language.Language, java.nio.file.Path)", + "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder addDefaultModel(java.nio.file.Path)", + "public java.util.Map getModels()", + "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder setCollapseUnknowns(boolean)", + "public boolean getCollapseUnknowns()", + "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder setScoring(com.yahoo.language.sentencepiece.Scoring)", + "public com.yahoo.language.sentencepiece.Scoring getScoring()", + "public com.yahoo.language.sentencepiece.SentencePieceEncoder build()" + ], + "fields": [] + }, + "com.yahoo.language.sentencepiece.SentencePieceEncoder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.language.process.Segmenter", + "com.yahoo.language.process.Encoder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)", + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder)", + "public java.util.List segment(java.lang.String, com.yahoo.language.Language)", + "public java.util.List encode(java.lang.String, com.yahoo.language.Language)", + "public com.yahoo.tensor.Tensor encode(java.lang.String, com.yahoo.language.Language, com.yahoo.tensor.TensorType)", + "public java.lang.String normalize(java.lang.String)" + ], + "fields": [] + } +}
\ No newline at end of file diff --git a/linguistics-components/pom.xml b/linguistics-components/pom.xml new file mode 100644 index 00000000000..44e58fb7588 --- /dev/null +++ b/linguistics-components/pom.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>linguistics-components</artifactId> + <packaging>container-plugin</packaging> + <version>7-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>linguistics</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <scope>provided</scope> + <classifier>no_aop</classifier> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.github.os72</groupId> + <artifactId>protoc-jar-maven-plugin</artifactId> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:rawtypes</arg> + <arg>-Xlint:unchecked</arg> + <arg>-Xlint:deprecation</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>abi-check-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Model.java index 74f300057dc..74f300057dc 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Model.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java index 2141505374c..2141505374c 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Scoring.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Scoring.java index 6c8560abee7..6c8560abee7 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Scoring.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Scoring.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java index 1659e3c0fa7..1659e3c0fa7 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java index b6659ebeaa3..b6659ebeaa3 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java index 782030a8e4d..782030a8e4d 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Trie.java index 8e7c2db2ed3..8e7c2db2ed3 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Trie.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/package-info.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/package-info.java index 4a8673705ec..3f97277c489 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/package-info.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2021 Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage @PublicApi package com.yahoo.language.sentencepiece; diff --git a/linguistics/src/main/protobuf/sentencepiece_model.proto b/linguistics-components/src/main/protobuf/sentencepiece_model.proto index 39626aede53..39626aede53 100644 --- a/linguistics/src/main/protobuf/sentencepiece_model.proto +++ b/linguistics-components/src/main/protobuf/sentencepiece_model.proto diff --git a/linguistics/src/main/resources/configdefinitions/sentence-piece.def b/linguistics-components/src/main/resources/configdefinitions/sentence-piece.def index b91c0c45dc4..b91c0c45dc4 100644 --- a/linguistics/src/main/resources/configdefinitions/sentence-piece.def +++ b/linguistics-components/src/main/resources/configdefinitions/sentence-piece.def diff --git a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java index edbbe21ec53..edbbe21ec53 100644 --- a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java diff --git a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java index d60d7386d4b..d60d7386d4b 100644 --- a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java diff --git a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java index 1ba7c9b472d..1ba7c9b472d 100644 --- a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java diff --git a/linguistics/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model b/linguistics-components/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model Binary files differindex 89f93ef3517..89f93ef3517 100644 --- a/linguistics/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model +++ b/linguistics-components/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model diff --git a/linguistics/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model b/linguistics-components/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model Binary files differindex 41c0688d9df..41c0688d9df 100644 --- a/linguistics/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model +++ b/linguistics-components/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index dbf4842ea1a..cfbf2abda1a 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -731,192 +731,5 @@ "public abstract java.lang.String accentDrop(java.lang.String, com.yahoo.language.Language)" ], "fields": [] - }, - "com.yahoo.language.sentencepiece.Scoring": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.language.sentencepiece.Scoring[] values()", - "public static com.yahoo.language.sentencepiece.Scoring valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.language.sentencepiece.Scoring highestScore", - "public static final enum com.yahoo.language.sentencepiece.Scoring fewestSegments" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Builder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder collapseUnknowns(boolean)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder scoring(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(java.util.List)", - "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", - "public final java.lang.String getDefMd5()", - "public final java.lang.String getDefName()", - "public final java.lang.String getDefNamespace()", - "public final boolean getApplyOnRestart()", - "public final void setApplyOnRestart(boolean)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig build()" - ], - "fields": [ - "public java.util.List model" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder language(java.lang.String)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder path(com.yahoo.config.FileReference)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model build()" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Model": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)", - "public java.lang.String language()", - "public java.nio.file.Path path()" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Producer": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Producer" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void getConfig(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum[] values()", - "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore", - "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring": { - "superClass": "com.yahoo.config.EnumNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)" - ], - "fields": [ - "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore", - "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig": { - "superClass": "com.yahoo.config.ConfigInstance", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public static java.lang.String getDefMd5()", - "public static java.lang.String getDefName()", - "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)", - "public boolean collapseUnknowns()", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum scoring()", - "public java.util.List model()", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model model(int)" - ], - "fields": [ - "public static final java.lang.String CONFIG_DEF_MD5", - "public static final java.lang.String CONFIG_DEF_NAME", - "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", - "public static final java.lang.String[] CONFIG_DEF_SCHEMA" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void addModel(com.yahoo.language.Language, java.nio.file.Path)", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder addDefaultModel(java.nio.file.Path)", - "public java.util.Map getModels()", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder setCollapseUnknowns(boolean)", - "public boolean getCollapseUnknowns()", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder setScoring(com.yahoo.language.sentencepiece.Scoring)", - "public com.yahoo.language.sentencepiece.Scoring getScoring()", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder build()" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceEncoder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.language.process.Segmenter", - "com.yahoo.language.process.Encoder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder)", - "public java.util.List segment(java.lang.String, com.yahoo.language.Language)", - "public java.util.List encode(java.lang.String, com.yahoo.language.Language)", - "public com.yahoo.tensor.Tensor encode(java.lang.String, com.yahoo.language.Language, com.yahoo.tensor.TensorType)", - "public java.lang.String normalize(java.lang.String)" - ], - "fields": [] } }
\ No newline at end of file diff --git a/linguistics/pom.xml b/linguistics/pom.xml index 221d7181616..0e5f9e15b85 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -15,10 +15,6 @@ <version>7-SNAPSHOT</version> <dependencies> <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> @@ -94,6 +94,7 @@ <module>jdisc_jetty</module> <module>jrt</module> <module>linguistics</module> + <module>linguistics-components</module> <module>logd</module> <module>logserver</module> <module>messagebus</module> diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java index ed7d30c476f..dc50db1e598 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java @@ -37,7 +37,7 @@ public class LocalFileDb implements FileAcquirer, FileRegistry { synchronized (this) { File file = fileReferenceToFile.get(reference); if (file == null) { - throw new RuntimeException("Invalid file reference " + reference); + return new File(reference.value()); // Downloaded file reference: Will (hopefully) be resolved client side } return file; } |