1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package ai.vespa.models.evaluation;
import com.yahoo.config.subscription.ConfigGetter;
import com.yahoo.config.subscription.FileSource;
import com.yahoo.path.Path;
import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
import org.junit.Test;
import java.io.File;
import java.util.Map;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* Tests instantiating models from rank-profiles configs.
*
* @author bratseth
*/
public class RankProfilesImporterTest {
@Test
public void testImportingModels() {
Map<String, Model> models = createModels("src/test/resources/config/models/");
assertEquals(4, models.size());
// TODO: When we get type information in Models, replace the evaluator.context().names() check below by that
{
Model xgboost = models.get("xgboost_2_2");
assertFunction("xgboost_2_2",
"(optimized sum of condition trees of size 192 bytes)",
xgboost);
FunctionEvaluator evaluator = xgboost.evaluatorOf();
assertEquals("f109, f29, f56, f60", evaluator.context().names().stream().sorted().collect(Collectors.joining(", ")));
}
{
Model onnxMnistSoftmax = models.get("mnist_softmax");
assertFunction("default.add",
"join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))",
onnxMnistSoftmax);
assertEquals("tensor(d1[10],d2[784])",
onnxMnistSoftmax.evaluatorOf("default.add").context().get("constant(mnist_softmax_Variable)").type().toString());
FunctionEvaluator evaluator = onnxMnistSoftmax.evaluatorOf(); // Verify exactly one output available
assertEquals("Placeholder, constant(mnist_softmax_Variable), constant(mnist_softmax_Variable_1)", evaluator.context().names().stream().sorted().collect(Collectors.joining(", ")));
}
{
Model tfMnistSoftmax = models.get("mnist_softmax_saved");
assertFunction("serving_default.y",
"join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))",
tfMnistSoftmax);
FunctionEvaluator evaluator = tfMnistSoftmax.evaluatorOf(); // Verify exactly one output available
assertEquals("Placeholder, constant(mnist_softmax_saved_layer_Variable_1_read), constant(mnist_softmax_saved_layer_Variable_read)", evaluator.context().names().stream().sorted().collect(Collectors.joining(", ")));
}
{
Model tfMnist = models.get("mnist_saved");
assertFunction("serving_default.y",
"join(reduce(join(map(join(reduce(join(join(join(rankingExpression(imported_ml_macro_mnist_saved_dnn_hidden1_add), 0.009999999776482582, f(a,b)(a * b)), rankingExpression(imported_ml_macro_mnist_saved_dnn_hidden1_add), f(a,b)(max(a,b))), constant(mnist_saved_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(mnist_saved_dnn_hidden2_bias_read), f(a,b)(a + b)), f(a)(1.050701 * if (a >= 0, a, 1.673263 * (exp(a) - 1)))), constant(mnist_saved_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(mnist_saved_dnn_outputs_bias_read), f(a,b)(a + b))",
tfMnist);
// Macro:
assertFunction("imported_ml_macro_mnist_saved_dnn_hidden1_add",
"join(reduce(join(rename(input, (d0, d1), (d0, d4)), constant(mnist_saved_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(mnist_saved_dnn_hidden1_bias_read), f(a,b)(a + b))",
tfMnist);
FunctionEvaluator evaluator = tfMnist.evaluatorOf("serving_default"); // TODO: Macro is offered as an alternative output currently, so need to specify argument
assertEquals("constant(mnist_saved_dnn_hidden1_bias_read), constant(mnist_saved_dnn_hidden1_weights_read), constant(mnist_saved_dnn_hidden2_bias_read), constant(mnist_saved_dnn_hidden2_weights_read), constant(mnist_saved_dnn_outputs_bias_read), constant(mnist_saved_dnn_outputs_weights_read), input, rankingExpression(imported_ml_macro_mnist_saved_dnn_hidden1_add)", evaluator.context().names().stream().sorted().collect(Collectors.joining(", ")));
}
}
@Test
public void testImportingRankExpressions() {
Map<String, Model> models = createModels("src/test/resources/config/rankexpression/");
assertEquals(18, models.size());
Model macros = models.get("macros");
assertEquals("macros", macros.name());
assertEquals(4, macros.functions().size());
assertFunction("fourtimessum", "4 * (var1 + var2)", macros);
assertFunction("firstphase", "match + fieldMatch(title) + rankingExpression(myfeature)", macros);
assertFunction("secondphase", "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86)", macros);
assertFunction("myfeature",
"70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + " +
"30 * pow(0 - fieldMatch(description).earliness,2)",
macros);
assertEquals(4, macros.referencedFunctions().size());
assertBoundFunction("rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86)",
"4 * (match + rankBoost)", macros);
}
private void assertFunction(String name, String expression, Model model) {
assertNotNull("Model is present in config", model);
ExpressionFunction function = model.function(name);
assertNotNull("Function '" + name + "' is in " + model, function);
assertEquals(name, function.getName());
assertEquals(expression, function.getBody().getRoot().toString());
}
private void assertBoundFunction(String name, String expression, Model model) {
ExpressionFunction function = model.referencedFunctions().get(FunctionReference.fromSerial(name).get());
assertNotNull("Function '" + name + "' is present", function);
assertEquals(name, function.getName());
assertEquals(expression, function.getBody().getRoot().toString());
}
private Map<String, Model> createModels(String path) {
Path configDir = Path.fromString(path);
RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()),
RankProfilesConfig.class).getConfig("");
RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()),
RankingConstantsConfig.class).getConfig("");
return new RankProfilesConfigImporter().importFrom(config, constantsConfig);
}
}
|
