1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.process;
import com.yahoo.language.Language;
import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.TensorType;
import java.util.List;
import java.util.Map;
/**
* An embedder converts a text string to a tensor
*
* @author bratseth
*/
public interface Embedder {
/** Name of embedder when none is explicity given */
String defaultEmbedderId = "default";
/** An instance of this which throws IllegalStateException if attempted used */
Embedder throwsOnUse = new FailingEmbedder();
/** Returns this embedder instance as a map with the default embedder name */
default Map<String, Embedder> asMap() {
return asMap(defaultEmbedderId);
}
/** Returns this embedder instance as a map with the given name */
default Map<String, Embedder> asMap(String name) {
return Map.of(name, this);
}
/**
* Converts text into a list of token id's (a vector embedding)
*
* @param text the text to embed
* @param context the context which may influence an embedder's behavior
* @return the text embedded as a list of token ids
* @throws IllegalArgumentException if the language is not supported by this embedder
*/
List<Integer> embed(String text, Context context);
/**
* Converts the list of token id's into a text. The opposite operation of embed.
*
* @param tokens the list of tokens to decode to a string
* @param context the context which specifies the language used to select a model
* @return the string formed by decoding the tokens back to their string repreesentation
*/
default String decode(List<Integer> tokens, Context context) {
throw new UnsupportedOperationException("Decode is not implemented");
}
/**
* Converts text into tokens in a tensor.
* The information contained in the embedding may depend on the tensor type.
*
* @param text the text to embed
* @param context the context which may influence an embedder's behavior
* @param tensorType the type of the tensor to be returned
* @return the tensor embedding of the text, as the specified tensor type
* @throws IllegalArgumentException if the language or tensor type is not supported by this embedder
*/
Tensor embed(String text, Context context, TensorType tensorType);
class Context {
private Language language = Language.UNKNOWN;
private String destination;
public Context(String destination) {
this.destination = destination;
}
/** Returns the language of the text, or UNKNOWN (default) to use a language independent embedding */
public Language getLanguage() { return language; }
/** Sets the language of the text, or UNKNOWN to use language independent embedding */
public Context setLanguage(Language language) {
this.language = language != null ? language : Language.UNKNOWN;
return this;
}
/**
* Returns the name of the recipient of this tensor.
*
* This is either a query feature name
* ("query(feature)"), or a schema and field name concatenated by a dot ("schema.field").
* This cannot be null.
*/
public String getDestination() { return destination; }
/**
* Sets the name of the recipient of this tensor.
*
* This is either a query feature name
* ("query(feature)"), or a schema and field name concatenated by a dot ("schema.field").
*/
public Context setDestination(String destination) {
this.destination = destination;
return this;
}
}
class FailingEmbedder implements Embedder {
private final String message;
public FailingEmbedder() {
this("No embedder has been configured");
}
public FailingEmbedder(String message) {
this.message = message;
}
@Override
public List<Integer> embed(String text, Context context) {
throw new IllegalStateException(message);
}
@Override
public Tensor embed(String text, Context context, TensorType tensorType) {
throw new IllegalStateException(message);
}
}
}
|
