diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-09-10 09:27:22 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@oath.com> | 2018-09-10 09:27:22 +0200 |
commit | 80476877f811f0a90cc1e692d56b5fbd11fbe76c (patch) | |
tree | 8b6451c43e3d90bc0ddf9fbcf268dcbd6d1c2a05 /model-evaluation/src/main | |
parent | 48f9a4071b5f6f75114cd1b94eb09bec36f77061 (diff) |
Use fileAcquirer
Diffstat (limited to 'model-evaluation/src/main')
-rw-r--r-- | model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java | 7 | ||||
-rw-r--r-- | model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java | 22 |
2 files changed, 20 insertions, 9 deletions
diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index b507c5d7f08..a0b859bf930 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -5,6 +5,7 @@ import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; @@ -24,8 +25,10 @@ public class ModelsEvaluator extends AbstractComponent { private final ImmutableMap<String, Model> models; @Inject - public ModelsEvaluator(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { - this(new RankProfilesConfigImporter().importFrom(config, constantsConfig)); + public ModelsEvaluator(RankProfilesConfig config, + RankingConstantsConfig constantsConfig, + FileAcquirer fileAcquirer) { + this(new RankProfilesConfigImporter(fileAcquirer).importFrom(config, constantsConfig)); } public ModelsEvaluator(Map<String, Model> models) { diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index 00f60654318..87ac53488db 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import com.yahoo.config.FileReference; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; @@ -11,7 +13,6 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.defaults.Defaults; import java.io.File; import java.io.IOException; @@ -21,7 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.logging.Logger; +import java.util.concurrent.TimeUnit; /** * Converts RankProfilesConfig instances to RankingExpressions for evaluation. @@ -31,6 +32,12 @@ import java.util.logging.Logger; */ public class RankProfilesConfigImporter { + private final FileAcquirer fileAcquirer; + + public RankProfilesConfigImporter(FileAcquirer fileAcquirer) { + this.fileAcquirer = fileAcquirer; + } + /** * Returns a map of the models contained in this config, indexed on name. * The map is modifiable and owned by the caller. @@ -107,16 +114,14 @@ public class RankProfilesConfigImporter { constants.add(new Constant(constantConfig.name(), readTensorFromFile(constantConfig.name(), TensorType.fromSpec(constantConfig.type()), - constantConfig.fileref().value()))); + constantConfig.fileref()))); } return constants; } - protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + protected Tensor readTensorFromFile(String name, TensorType type, FileReference fileReference) { try { - File dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference); - File file = dir.listFiles()[0]; // directory contains one file having the original name - + File file = fileAcquirer.waitFor(fileReference, 7, TimeUnit.DAYS); if (file.getName().endsWith(".tbf")) return TypedBinaryFormat.decode(Optional.of(type), GrowableByteBuffer.wrap(IOUtils.readFileBytes(file))); @@ -125,6 +130,9 @@ public class RankProfilesConfigImporter { file + " for constant " + name); // TODO: Support json and json.lz4 } + catch (InterruptedException e) { + throw new IllegalStateException("Gave up waiting for constant " + name); + } catch (IOException e) { throw new UncheckedIOException(e); } |