summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2021-06-01 17:00:44 +0200
committerHarald Musum <musum@verizonmedia.com>2021-06-01 17:00:44 +0200
commit37a6f21b70ec39bdc6d2ea2c5d5b1359748fa92e (patch)
treead378d7a5e3e310993f6ce5633c12d319c51a290
parent2f67aed1f678d626ed9694a94041437c37e7d688 (diff)
Write application package schemas to schemas dir in ZooKeeper
-rw-r--r--config-model/src/test/cfg/application/ml_models/schemas/test.sd (renamed from config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd)0
-rw-r--r--config-model/src/test/cfg/application/onnx/schemas/test.sd (renamed from config-model/src/test/cfg/application/onnx/searchdefinitions/test.sd)0
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java6
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java9
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java9
6 files changed, 11 insertions, 17 deletions
diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/schemas/test.sd
index cc73f2daff5..cc73f2daff5 100644
--- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd
+++ b/config-model/src/test/cfg/application/ml_models/schemas/test.sd
diff --git a/config-model/src/test/cfg/application/onnx/searchdefinitions/test.sd b/config-model/src/test/cfg/application/onnx/schemas/test.sd
index d49782ddf39..d49782ddf39 100644
--- a/config-model/src/test/cfg/application/onnx/searchdefinitions/test.sd
+++ b/config-model/src/test/cfg/application/onnx/schemas/test.sd
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
index 854c4d32a64..9e87b5509bc 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
@@ -32,8 +32,8 @@ public class MlModelsTest {
IOUtils.copy(appDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString());
IOUtils.copyDirectory(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(),
storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
- IOUtils.copyDirectory(appDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile(),
- storedAppDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile());
+ IOUtils.copyDirectory(appDir.append(ApplicationPackage.SCHEMAS_DIR).toFile(),
+ storedAppDir.append(ApplicationPackage.SCHEMAS_DIR).toFile());
ImportedModelTester storedTester = new ImportedModelTester("ml_models", storedAppDir);
verify(storedTester.createVespaModel());
}
@@ -61,7 +61,7 @@ public class MlModelsTest {
assertEquals(testProfile, b.toString());
}
- private final String testProfile =
+ private static final String testProfile =
"rankingExpression(Placeholder).rankingScript: attribute(argument)\n" +
"rankingExpression(Placeholder).type: tensor<float>(d0[1],d1[784])\n" +
"rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" +
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java
index 5dea4a04229..5c64dd9f8cd 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java
@@ -4,17 +4,13 @@ package com.yahoo.vespa.model.ml;
import ai.vespa.models.evaluation.FunctionEvaluator;
import ai.vespa.models.evaluation.Model;
import ai.vespa.models.evaluation.ModelsEvaluator;
-import ai.vespa.models.evaluation.RankProfilesConfigImporter;
-import ai.vespa.models.handler.ModelsEvaluationHandler;
import com.yahoo.component.ComponentId;
-import com.yahoo.config.FileReference;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.filedistribution.fileacquirer.FileAcquirer;
import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer;
import com.yahoo.io.IOUtils;
import com.yahoo.path.Path;
import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorType;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
@@ -31,7 +27,6 @@ import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
/**
@@ -55,8 +50,8 @@ public class StatelessOnnxEvaluationTest {
IOUtils.copy(appDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString());
IOUtils.copyDirectory(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(),
storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
- IOUtils.copyDirectory(appDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile(),
- storedAppDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile());
+ IOUtils.copyDirectory(appDir.append(ApplicationPackage.SCHEMAS_DIR).toFile(),
+ storedAppDir.append(ApplicationPackage.SCHEMAS_DIR).toFile());
ImportedModelTester storedTester = new ImportedModelTester("onnx_rt", storedAppDir);
assertModelEvaluation(storedTester.createVespaModel(), appDir);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
index 1ab99db8985..ca7489012e2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
@@ -94,9 +94,7 @@ public class ZooKeeperClient {
Collection<NamedReader> sds = app.getSearchDefinitions();
if (sds.isEmpty()) return;
- // TODO: Change to SCHEMAS_DIR after March 2020
- // TODO: When it does also check RankExpressionFile.sendTo
- Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR);
+ Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR);
configCurator.createNode(zkPath.getAbsolute());
// Ensures that ranking expressions and other files are also written
writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
index f01bf1fe6ef..48c47587c8b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
@@ -134,14 +134,15 @@ public class ZKApplicationPackage implements ApplicationPackage {
@Override
public List<NamedReader> searchDefinitionContents() {
List<NamedReader> schemas = new ArrayList<>();
- for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) {
- if (sd.endsWith(SD_NAME_SUFFIX))
- schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd))));
- }
for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) {
if (sd.endsWith(SD_NAME_SUFFIX))
schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd))));
}
+ // TODO: Remove when everything is written to SCHEMAS_DIR (July 2021)
+ for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) {
+ if (sd.endsWith(SD_NAME_SUFFIX))
+ schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd))));
+ }
return schemas;
}