diff options
author | gjoranv <gv@verizonmedia.com> | 2022-02-01 01:17:10 +0100 |
---|---|---|
committer | gjoranv <gv@verizonmedia.com> | 2022-02-01 01:17:10 +0100 |
commit | cd1129733dd5353e26b60215185b3f119c43e022 (patch) | |
tree | c8205b920e3f9726258321026a4ec4e9998eacb9 /model-evaluation | |
parent | f62a05ea11eeef5e6c7e1605bcb9968a4325e7d8 (diff) |
Replace usage of deprecated ConfigGetter method and FileSource.
Diffstat (limited to 'model-evaluation')
5 files changed, 63 insertions, 73 deletions
diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index 3f1a86e083b..a36215e005f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -1,30 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; -import com.yahoo.config.FileReference; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.FileSource; -import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; -import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.serialization.TypedBinaryFormat; -import com.yahoo.text.Utf8; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; -import java.io.IOException; import java.util.Map; -import java.util.Optional; -import java.util.logging.Logger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -45,17 +32,19 @@ public class ModelTester { public Map<String, Model> models() { return models; } private static Map<String, Model> createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - RankingExpressionsConfig expresionsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-expressions.cfg").toFile()), - RankingExpressionsConfig.class).getConfig(""); - OnnxModelsConfig onnxModelsConfig = new ConfigGetter<>(new FileSource(configDir.append("onnx-models.cfg").toFile()), - OnnxModelsConfig.class).getConfig(""); + + RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId(path, "rank-profiles.cfg")); + RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId(path, "ranking-constants.cfg")); + RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId(path, "ranking-expressions.cfg")); + OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId(path, "onnx-models.cfg")); + + return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"), MockFileAcquirer.returnFile(null)) - .importFrom(config, constantsConfig, expresionsConfig, onnxModelsConfig); + .importFrom(config, constantsConfig, expressionsConfig, onnxModelsConfig); + } + + private static String fileConfigId(String path, String filename) { + return "file:" + path + filename; } public ExpressionFunction assertFunction(String name, String expression, Model model) { diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index 4cb52216137..540c534925e 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -28,10 +28,11 @@ import static org.junit.Assert.assertTrue; public class ModelsEvaluatorTest { private static final double delta = 0.00000000001; + private static final String CONFIG_DIR = "src/test/resources/config/rankexpression/"; @Test public void testEvaluationDependingFunctionTakingArguments() { - ModelsEvaluator models = createModels("src/test/resources/config/rankexpression/"); + ModelsEvaluator models = createModels(); FunctionEvaluator function = models.evaluatorOf("macros", "secondphase"); function.bind("match", 3); function.bind("rankBoost", 5); @@ -41,7 +42,7 @@ public class ModelsEvaluatorTest { /** Tests a function defined as 4 * (var1 + var2) */ @Test public void testSettingMissingValue() { - ModelsEvaluator models = createModels("src/test/resources/config/rankexpression/"); + ModelsEvaluator models = createModels(); { FunctionEvaluator function = models.evaluatorOf("macros", "secondphase"); @@ -127,18 +128,18 @@ public class ModelsEvaluatorTest { // TODO: Test argument-less function // TODO: Test with nested functions - private ModelsEvaluator createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - RankingExpressionsConfig expressionsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-expressions.cfg").toFile()), - RankingExpressionsConfig.class).getConfig(""); - OnnxModelsConfig onnxModelsConfig = new ConfigGetter<>(new FileSource(configDir.append("onnx-models.cfg").toFile()), - OnnxModelsConfig.class).getConfig(""); - return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"), MockFileAcquirer.returnFile(null)), + private ModelsEvaluator createModels() { + RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); + RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); + RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); + OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); + + return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(CONFIG_DIR).append("constants"), MockFileAcquirer.returnFile(null)), config, constantsConfig, expressionsConfig, onnxModelsConfig); } + private static String fileConfigId(String filename) { + return "file:" + CONFIG_DIR + filename; + } + } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java index a15c35fe854..c5559d9bed5 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java @@ -26,10 +26,11 @@ import static org.junit.Assert.assertTrue; public class OnnxEvaluatorTest { private static final double delta = 0.00000000001; + private static final String CONFIG_DIR = "src/test/resources/config/onnx/"; @Test public void testOnnxEvaluation() { - ModelsEvaluator models = createModels("src/test/resources/config/onnx/"); + ModelsEvaluator models = createModels(); assertTrue(models.models().containsKey("add_mul")); assertTrue(models.models().containsKey("one_layer")); @@ -49,24 +50,23 @@ public class OnnxEvaluatorTest { assertEquals(function.evaluate(), Tensor.from("tensor<float>(d0[2],d1[1]):[0.63931,0.67574]")); } - private ModelsEvaluator createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - RankingExpressionsConfig expressionsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-expressions.cfg").toFile()), - RankingExpressionsConfig.class).getConfig(""); - OnnxModelsConfig onnxModelsConfig = new ConfigGetter<>(new FileSource(configDir.append("onnx-models.cfg").toFile()), - OnnxModelsConfig.class).getConfig(""); + private ModelsEvaluator createModels() { + RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); + RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); + RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); + OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); Map<String, File> fileMap = new HashMap<>(); for (OnnxModelsConfig.Model onnxModel : onnxModelsConfig.model()) { - fileMap.put(onnxModel.fileref().value(), new File(path + onnxModel.fileref().value())); + fileMap.put(onnxModel.fileref().value(), new File(CONFIG_DIR + onnxModel.fileref().value())); } FileAcquirer fileAcquirer = MockFileAcquirer.returnFiles(fileMap); return new ModelsEvaluator(config, constantsConfig, expressionsConfig, onnxModelsConfig, fileAcquirer); } + private static String fileConfigId(String filename) { + return "file:" + CONFIG_DIR + filename; + } + } diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 33e56d5d465..bb442d76763 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -4,7 +4,6 @@ package ai.vespa.models.handler; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.evaluation.RankProfilesConfigImporterWithMockedConstants; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; @@ -21,11 +20,12 @@ import java.util.Map; public class ModelsEvaluationHandlerTest { + private static final String MODELS_DIR = "src/test/resources/config/models/"; private static HandlerTester handler; @BeforeClass static public void setUp() { - handler = new HandlerTester(createModels("src/test/resources/config/models/")); + handler = new HandlerTester(createModels()); } @Test @@ -251,20 +251,20 @@ public class ModelsEvaluationHandlerTest { handler.assertResponse(url, properties, 200, expected); } - static private ModelsEvaluator createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - RankingExpressionsConfig expressionsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-expressions.cfg").toFile()), - RankingExpressionsConfig.class).getConfig(""); - OnnxModelsConfig onnxModelsConfig = new ConfigGetter<>(new FileSource(configDir.append("onnx-models.cfg").toFile()), - OnnxModelsConfig.class).getConfig(""); - return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"), MockFileAcquirer.returnFile(null)), + static private ModelsEvaluator createModels() { + RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); + RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); + RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); + OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); + + return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(MODELS_DIR).append("constants"), MockFileAcquirer.returnFile(null)), config, constantsConfig, expressionsConfig, onnxModelsConfig); } + private static String fileConfigId(String filename) { + return "file:" + MODELS_DIR + filename; + } + private String inputTensor() { Tensor.Builder b = Tensor.Builder.of(TensorType.fromSpec("tensor(d0[],d1[784])")); for (int i = 0; i < 784; i++) diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java index 6014bd7c7ef..ccd303990c8 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java @@ -22,10 +22,11 @@ import java.util.Map; public class OnnxEvaluationHandlerTest { private static HandlerTester handler; + private static final String CONFIG_DIR = "src/test/resources/config/onnx/"; @BeforeClass static public void setUp() { - handler = new HandlerTester(createModels("src/test/resources/config/onnx/")); + handler = new HandlerTester(createModels()); } @Test @@ -114,24 +115,23 @@ public class OnnxEvaluationHandlerTest { handler.assertResponse(url, properties, 200, expected); } - static private ModelsEvaluator createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - RankingExpressionsConfig expressionsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-expressions.cfg").toFile()), - RankingExpressionsConfig.class).getConfig(""); - OnnxModelsConfig onnxModelsConfig = new ConfigGetter<>(new FileSource(configDir.append("onnx-models.cfg").toFile()), - OnnxModelsConfig.class).getConfig(""); + static private ModelsEvaluator createModels() { + RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); + RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); + RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); + OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); Map<String, File> fileMap = new HashMap<>(); for (OnnxModelsConfig.Model onnxModel : onnxModelsConfig.model()) { - fileMap.put(onnxModel.fileref().value(), new File(path + onnxModel.fileref().value())); + fileMap.put(onnxModel.fileref().value(), new File(CONFIG_DIR + onnxModel.fileref().value())); } FileAcquirer fileAcquirer = MockFileAcquirer.returnFiles(fileMap); return new ModelsEvaluator(config, constantsConfig, expressionsConfig, onnxModelsConfig, fileAcquirer); } + private static String fileConfigId(String filename) { + return "file:" + CONFIG_DIR + filename; + } + } |