diff options
author | Harald Musum <musum@verizonmedia.com> | 2021-06-01 17:00:44 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2021-06-01 17:00:44 +0200 |
commit | 37a6f21b70ec39bdc6d2ea2c5d5b1359748fa92e (patch) | |
tree | ad378d7a5e3e310993f6ce5633c12d319c51a290 | |
parent | 2f67aed1f678d626ed9694a94041437c37e7d688 (diff) |
Write application package schemas to schemas dir in ZooKeeper
-rw-r--r-- | config-model/src/test/cfg/application/ml_models/schemas/test.sd (renamed from config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd) | 0 | ||||
-rw-r--r-- | config-model/src/test/cfg/application/onnx/schemas/test.sd (renamed from config-model/src/test/cfg/application/onnx/searchdefinitions/test.sd) | 0 | ||||
-rw-r--r-- | config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java | 6 | ||||
-rw-r--r-- | config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java | 9 | ||||
-rw-r--r-- | configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java | 4 | ||||
-rw-r--r-- | configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java | 9 |
6 files changed, 11 insertions, 17 deletions
diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/schemas/test.sd index cc73f2daff5..cc73f2daff5 100644 --- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/ml_models/schemas/test.sd diff --git a/config-model/src/test/cfg/application/onnx/searchdefinitions/test.sd b/config-model/src/test/cfg/application/onnx/schemas/test.sd index d49782ddf39..d49782ddf39 100644 --- a/config-model/src/test/cfg/application/onnx/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/onnx/schemas/test.sd diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index 854c4d32a64..9e87b5509bc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -32,8 +32,8 @@ public class MlModelsTest { IOUtils.copy(appDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString()); IOUtils.copyDirectory(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - IOUtils.copyDirectory(appDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile(), - storedAppDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile()); + IOUtils.copyDirectory(appDir.append(ApplicationPackage.SCHEMAS_DIR).toFile(), + storedAppDir.append(ApplicationPackage.SCHEMAS_DIR).toFile()); ImportedModelTester storedTester = new ImportedModelTester("ml_models", storedAppDir); verify(storedTester.createVespaModel()); } @@ -61,7 +61,7 @@ public class MlModelsTest { assertEquals(testProfile, b.toString()); } - private final String testProfile = + private static final String testProfile = "rankingExpression(Placeholder).rankingScript: attribute(argument)\n" + "rankingExpression(Placeholder).type: tensor<float>(d0[1],d1[784])\n" + "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java index 5dea4a04229..5c64dd9f8cd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java @@ -4,17 +4,13 @@ package com.yahoo.vespa.model.ml; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; -import ai.vespa.models.evaluation.RankProfilesConfigImporter; -import ai.vespa.models.handler.ModelsEvaluationHandler; import com.yahoo.component.ComponentId; -import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; @@ -31,7 +27,6 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -55,8 +50,8 @@ public class StatelessOnnxEvaluationTest { IOUtils.copy(appDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString()); IOUtils.copyDirectory(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - IOUtils.copyDirectory(appDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile(), - storedAppDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile()); + IOUtils.copyDirectory(appDir.append(ApplicationPackage.SCHEMAS_DIR).toFile(), + storedAppDir.append(ApplicationPackage.SCHEMAS_DIR).toFile()); ImportedModelTester storedTester = new ImportedModelTester("onnx_rt", storedAppDir); assertModelEvaluation(storedTester.createVespaModel(), appDir); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 1ab99db8985..ca7489012e2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -94,9 +94,7 @@ public class ZooKeeperClient { Collection<NamedReader> sds = app.getSearchDefinitions(); if (sds.isEmpty()) return; - // TODO: Change to SCHEMAS_DIR after March 2020 - // TODO: When it does also check RankExpressionFile.sendTo - Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR); + Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR); configCurator.createNode(zkPath.getAbsolute()); // Ensures that ranking expressions and other files are also written writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index f01bf1fe6ef..48c47587c8b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -134,14 +134,15 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> searchDefinitionContents() { List<NamedReader> schemas = new ArrayList<>(); - for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) { - if (sd.endsWith(SD_NAME_SUFFIX)) - schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd)))); - } for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) { if (sd.endsWith(SD_NAME_SUFFIX)) schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd)))); } + // TODO: Remove when everything is written to SCHEMAS_DIR (July 2021) + for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) { + if (sd.endsWith(SD_NAME_SUFFIX)) + schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd)))); + } return schemas; } |