diff options
Diffstat (limited to 'linguistics-components')
25 files changed, 29 insertions, 30 deletions
diff --git a/linguistics-components/CMakeLists.txt b/linguistics-components/CMakeLists.txt index a26095a4649..bb3ba25170c 100644 --- a/linguistics-components/CMakeLists.txt +++ b/linguistics-components/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_jar(linguistics-components-jar-with-dependencies.jar) diff --git a/linguistics-components/pom.xml b/linguistics-components/pom.xml index 68b0437ac3f..ef1d299a6ec 100644 --- a/linguistics-components/pom.xml +++ b/linguistics-components/pom.xml @@ -1,5 +1,5 @@ <?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> @@ -21,7 +21,6 @@ <dependency> <groupId>ai.djl.huggingface</groupId> <artifactId>tokenizers</artifactId> - <version>0.23.0</version> <exclusions> <exclusion> <groupId>com.google.code.gson</groupId> diff --git a/linguistics-components/src/main/java/com/yahoo/language/huggingface/Encoding.java b/linguistics-components/src/main/java/com/yahoo/language/huggingface/Encoding.java index 107900ff73c..f5804419387 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/huggingface/Encoding.java +++ b/linguistics-components/src/main/java/com/yahoo/language/huggingface/Encoding.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.huggingface; diff --git a/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java b/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java index a8302aac25e..a805fc79a64 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java +++ b/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.huggingface; @@ -185,4 +185,4 @@ public class HuggingFaceTokenizer extends AbstractComponent implements Embedder, public HuggingFaceTokenizer build() { return new HuggingFaceTokenizer(this); } } -}
\ No newline at end of file +} diff --git a/linguistics-components/src/main/java/com/yahoo/language/huggingface/ModelInfo.java b/linguistics-components/src/main/java/com/yahoo/language/huggingface/ModelInfo.java index 4b30b1f0435..4db2c7e692c 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/huggingface/ModelInfo.java +++ b/linguistics-components/src/main/java/com/yahoo/language/huggingface/ModelInfo.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.huggingface; diff --git a/linguistics-components/src/main/java/com/yahoo/language/huggingface/package-info.java b/linguistics-components/src/main/java/com/yahoo/language/huggingface/package-info.java index 7cec01ffed6..346c346d7f6 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/huggingface/package-info.java +++ b/linguistics-components/src/main/java/com/yahoo/language/huggingface/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * @author bjorncs @@ -6,4 +6,4 @@ @ExportPackage package com.yahoo.language.huggingface; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Model.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Model.java index 28a1b9d2930..b31f56c626b 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Model.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Model.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.sentencepiece; import com.yahoo.io.IOUtils; diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java index 2141505374c..5d7bf24c475 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.sentencepiece; /** diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Scoring.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Scoring.java index 6c8560abee7..d9f13ac7a54 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Scoring.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Scoring.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.sentencepiece; /** diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java index 1659e3c0fa7..4c2441ec5d9 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.sentencepiece; /** diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java index b4d542b0d82..9105520bd86 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.sentencepiece; import com.yahoo.api.annotations.Beta; diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java index 782030a8e4d..64ad441b280 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.sentencepiece; /** diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Trie.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Trie.java index 8e7c2db2ed3..98249071147 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Trie.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Trie.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.sentencepiece; import java.util.HashMap; diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/package-info.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/package-info.java index 4a8673705ec..d94699b62e6 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/package-info.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage @PublicApi package com.yahoo.language.sentencepiece; diff --git a/linguistics-components/src/main/java/com/yahoo/language/tools/Embed.java b/linguistics-components/src/main/java/com/yahoo/language/tools/Embed.java index 401347cc452..affdb9a868a 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/tools/Embed.java +++ b/linguistics-components/src/main/java/com/yahoo/language/tools/Embed.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.tools; import com.yahoo.language.process.Embedder; diff --git a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/Model.java b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/Model.java index 0643eee7094..c8c820e9f8a 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/Model.java +++ b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/Model.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.wordpiece; import com.yahoo.collections.Tuple2; diff --git a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java index 55a0411cd8a..d8922489a9f 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java +++ b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.wordpiece; import com.yahoo.component.annotation.Inject; diff --git a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/package-info.java b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/package-info.java index 0bbb6f001f5..4077f400aab 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/package-info.java +++ b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage @PublicApi package com.yahoo.language.wordpiece; diff --git a/linguistics-components/src/main/resources/configdefinitions/language.sentencepiece.sentence-piece.def b/linguistics-components/src/main/resources/configdefinitions/language.sentencepiece.sentence-piece.def index 16ada78688a..d470c08099d 100644 --- a/linguistics-components/src/main/resources/configdefinitions/language.sentencepiece.sentence-piece.def +++ b/linguistics-components/src/main/resources/configdefinitions/language.sentencepiece.sentence-piece.def @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Configures com.yahoo.language.sentencepiece.SentencePieceEmbedder @@ -15,4 +15,4 @@ scoring enum { highestScore, fewestSegments } default=fewestSegments # Use "unknown" for models to be used with any language. model[].language string # The path to the model relative to the application package root -model[].path path
\ No newline at end of file +model[].path path diff --git a/linguistics-components/src/main/resources/configdefinitions/language.wordpiece.word-piece.def b/linguistics-components/src/main/resources/configdefinitions/language.wordpiece.word-piece.def index 08592250eb5..1d70255d97e 100644 --- a/linguistics-components/src/main/resources/configdefinitions/language.wordpiece.word-piece.def +++ b/linguistics-components/src/main/resources/configdefinitions/language.wordpiece.word-piece.def @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Configures com.yahoo.language.wordpiece.WordPieceEmbedder diff --git a/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java b/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java index f727252cccb..2974c8e0e8e 100644 --- a/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.huggingface; @@ -138,4 +138,4 @@ class HuggingFaceTokenizerTest { return destination; } -}
\ No newline at end of file +} diff --git a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java index 19cb2267655..d5b1f59cb20 100644 --- a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.sentencepiece; diff --git a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java index daa31f8773b..da0befb1696 100644 --- a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.sentencepiece; diff --git a/linguistics-components/src/test/java/com/yahoo/language/tools/EmbedderTester.java b/linguistics-components/src/test/java/com/yahoo/language/tools/EmbedderTester.java index a403b1ee943..5af759958b4 100644 --- a/linguistics-components/src/test/java/com/yahoo/language/tools/EmbedderTester.java +++ b/linguistics-components/src/test/java/com/yahoo/language/tools/EmbedderTester.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.tools; diff --git a/linguistics-components/src/test/java/com/yahoo/language/wordpiece/WordPieceEmbedderTest.java b/linguistics-components/src/test/java/com/yahoo/language/wordpiece/WordPieceEmbedderTest.java index 13e0cbce10d..9191b4add00 100644 --- a/linguistics-components/src/test/java/com/yahoo/language/wordpiece/WordPieceEmbedderTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/wordpiece/WordPieceEmbedderTest.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.wordpiece; import com.yahoo.config.FileReference; |