diff options
11 files changed, 60 insertions, 34 deletions
diff --git a/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java index 96886941fe1..ad2f62b7dc3 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java @@ -4,7 +4,10 @@ package com.yahoo.config.model; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.evaluation.RankProfilesConfigImporter; +import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; @@ -77,7 +80,7 @@ public class ModelEvaluationTest { assertTrue(modelNames.contains("mnist_softmax")); assertTrue(modelNames.contains("mnist_softmax_saved")); - ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter() + ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter(MockFileAcquirer.returnFile(null)) .importFrom(config, constantsConfig)); assertEquals(4, evaluator.models().size()); @@ -114,7 +117,11 @@ public class ModelEvaluationTest { // We don't have function file distribution so just return empty tensor constants private static class ToleratingMissingConstantFilesRankProfilesConfigImporter extends RankProfilesConfigImporter { - protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + public ToleratingMissingConstantFilesRankProfilesConfigImporter(FileAcquirer fileAcquirer) { + super(fileAcquirer); + } + + protected Tensor readTensorFromFile(String name, TensorType type, FileReference fileReference) { return Tensor.from(type, "{}"); } diff --git a/container-core/src/main/java/com/yahoo/container/Container.java b/container-core/src/main/java/com/yahoo/container/Container.java index efe7c58563c..7e5ea7bd948 100755 --- a/container-core/src/main/java/com/yahoo/container/Container.java +++ b/container-core/src/main/java/com/yahoo/container/Container.java @@ -145,7 +145,7 @@ public class Container { /** * Only for internal use. */ - public void setCustomFileAcquirer(final FileAcquirer fileAcquirer) { + public void setCustomFileAcquirer(FileAcquirer fileAcquirer) { if (this.fileAcquirer != null) { throw new RuntimeException("Can't change file acquirer. Is " + this.fileAcquirer + " attempted to set to " + fileAcquirer); @@ -155,7 +155,7 @@ public class Container { setPathAcquirer(fileAcquirer); } - private static void setPathAcquirer(final FileAcquirer fileAcquirer) { + private static void setPathAcquirer(FileAcquirer fileAcquirer) { ConfigTransformer.setPathAcquirer(fileReference -> { try { return fileAcquirer.waitFor(fileReference, 15, TimeUnit.MINUTES).toPath(); diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java index c0a68086212..557f331395b 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java @@ -41,8 +41,7 @@ public class BundleLoader { initialBundles = Arrays.asList(osgi.getBundles()); } - private List<Bundle> obtainBundles(FileReference reference, FileAcquirer fileAcquirer) - throws InterruptedException { + private List<Bundle> obtainBundles(FileReference reference, FileAcquirer fileAcquirer) throws InterruptedException { File file = fileAcquirer.waitFor(reference, 7, TimeUnit.DAYS); return osgi.install(file.getAbsolutePath()); } diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java index 2d0ff0c62db..7b24a00cf60 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java @@ -36,14 +36,13 @@ import java.util.logging.Logger; public abstract class QueryRewriteSearcher extends Searcher { // Indicate whether rewriter is properly initiated - private boolean isOk = false; + private boolean isOk; protected final Logger logger = Logger.getLogger(QueryRewriteSearcher.class.getName()); // HashMap which store the rewriter dicts // It has the following format: - // HashMap<String(e.g. dictionary name, etc), - // Object(e.g. FSA, etc)>> + // HashMap<String(e.g. dictionary name, etc), Object(e.g. FSA, etc)>> protected HashMap<String, Object> rewriterDicts = new HashMap<>(); /** @@ -201,14 +200,14 @@ public abstract class QueryRewriteSearcher extends Searcher { "FSA file location for " + fsaName + ": " + fsaPath); // Retrieve FSA File handler - File fsaFile = null; - if(fileAcquirer!=null) { + File fsaFile; + if (fileAcquirer != null) { fsaFile = fileAcquirer.waitFor(fsaPath, 5, TimeUnit.MINUTES); - } else if(fileList!=null) { + } else { fsaFile = fileList.get(fsaName); } - if(fsaFile==null) { + if (fsaFile == null) { RewriterUtils.error(logger, "Error loading FSA dictionary file handler"); return false; } diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerFactory.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerFactory.java index d8ea45e716d..e8a3038639a 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerFactory.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerFactory.java @@ -4,10 +4,13 @@ package com.yahoo.filedistribution.fileacquirer; /** * Hides the real file acquirer type from 3rd party developers. * Not intended to be used by 3rd parties. + * * @author Tony Vaagenes */ public class FileAcquirerFactory { + public static FileAcquirer create(String configId) { return new FileAcquirerImpl(configId); } + } diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java index fca4b206fc9..ab0f7521e7e 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java @@ -23,12 +23,15 @@ import java.io.File; * @author Tony Vaagenes */ class FileAcquirerImpl implements FileAcquirer { + static final class FileDistributionErrorCode { + public static final int baseErrorCode = 0x10000; public static final int baseFileProviderErrorCode = baseErrorCode + 0x1000; public static final int fileReferenceDoesNotExists = baseFileProviderErrorCode; public static final int fileReferenceRemoved = fileReferenceDoesNotExists + 1; + } private static final Logger log = Logger.getLogger(FileAcquirerImpl.class.getName()); @@ -131,13 +134,10 @@ class FileAcquirerImpl implements FileAcquirer { * given file reference. File references are produced by the * config system. * - * @throws TimeoutException if the file or directory could not be - * retrieved in time. - * @throws FileReferenceDoesNotExistException if the file is no - * longer available (due to reloading of config). + * @throws TimeoutException if the file or directory could not be retrieved in time. + * @throws FileReferenceDoesNotExistException if the file is no longer available (due to reloading of config). */ - public File waitFor(FileReference fileReference, long timeout, TimeUnit timeUnit) - throws InterruptedException { + public File waitFor(FileReference fileReference, long timeout, TimeUnit timeUnit) throws InterruptedException { Timer timer = new Timer(timeout, timeUnit); do { Target target = connection.getTarget(timer); diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/MockFileAcquirer.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/MockFileAcquirer.java index 25732d2dcc8..1a8a05d0a53 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/MockFileAcquirer.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/MockFileAcquirer.java @@ -14,8 +14,9 @@ import java.util.concurrent.TimeUnit; * @author Tony Vaagenes */ public abstract class MockFileAcquirer implements FileAcquirer { + /** Creates a FileAcquirer that always returns the given file. **/ - public static FileAcquirer returnFile(final File file) { + public static FileAcquirer returnFile(File file) { return new MockFileAcquirer() { @Override public File waitFor(FileReference fileReference, @@ -26,7 +27,7 @@ public abstract class MockFileAcquirer implements FileAcquirer { } /** Creates a FileAcquirer that maps from fileReference.value to a file. **/ - public static FileAcquirer returnFiles(final Map<String, File> files) { + public static FileAcquirer returnFiles(Map<String, File> files) { return new MockFileAcquirer() { @Override public File waitFor(FileReference fileReference, @@ -60,4 +61,5 @@ public abstract class MockFileAcquirer implements FileAcquirer { @Override public void shutdown() {} + } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index b507c5d7f08..a0b859bf930 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -5,6 +5,7 @@ import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; @@ -24,8 +25,10 @@ public class ModelsEvaluator extends AbstractComponent { private final ImmutableMap<String, Model> models; @Inject - public ModelsEvaluator(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { - this(new RankProfilesConfigImporter().importFrom(config, constantsConfig)); + public ModelsEvaluator(RankProfilesConfig config, + RankingConstantsConfig constantsConfig, + FileAcquirer fileAcquirer) { + this(new RankProfilesConfigImporter(fileAcquirer).importFrom(config, constantsConfig)); } public ModelsEvaluator(Map<String, Model> models) { diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index 00f60654318..87ac53488db 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import com.yahoo.config.FileReference; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; @@ -11,7 +13,6 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.defaults.Defaults; import java.io.File; import java.io.IOException; @@ -21,7 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.logging.Logger; +import java.util.concurrent.TimeUnit; /** * Converts RankProfilesConfig instances to RankingExpressions for evaluation. @@ -31,6 +32,12 @@ import java.util.logging.Logger; */ public class RankProfilesConfigImporter { + private final FileAcquirer fileAcquirer; + + public RankProfilesConfigImporter(FileAcquirer fileAcquirer) { + this.fileAcquirer = fileAcquirer; + } + /** * Returns a map of the models contained in this config, indexed on name. * The map is modifiable and owned by the caller. @@ -107,16 +114,14 @@ public class RankProfilesConfigImporter { constants.add(new Constant(constantConfig.name(), readTensorFromFile(constantConfig.name(), TensorType.fromSpec(constantConfig.type()), - constantConfig.fileref().value()))); + constantConfig.fileref()))); } return constants; } - protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + protected Tensor readTensorFromFile(String name, TensorType type, FileReference fileReference) { try { - File dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference); - File file = dir.listFiles()[0]; // directory contains one file having the original name - + File file = fileAcquirer.waitFor(fileReference, 7, TimeUnit.DAYS); if (file.getName().endsWith(".tbf")) return TypedBinaryFormat.decode(Optional.of(type), GrowableByteBuffer.wrap(IOUtils.readFileBytes(file))); @@ -125,6 +130,9 @@ public class RankProfilesConfigImporter { file + " for constant " + name); // TODO: Support json and json.lz4 } + catch (InterruptedException e) { + throw new IllegalStateException("Gave up waiting for constant " + name); + } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index 0a400f7126f..0aceaccc3e0 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -1,8 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import com.yahoo.config.FileReference; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.FileSource; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -42,7 +45,7 @@ public class ModelTester { RankProfilesConfig.class).getConfig(""); RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), RankingConstantsConfig.class).getConfig(""); - return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants")) + return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"), MockFileAcquirer.returnFile(null)) .importFrom(config, constantsConfig); } @@ -68,12 +71,13 @@ public class ModelTester { private final Path constantsPath; - public RankProfilesConfigImporterWithMockedConstants(Path constantsPath) { + public RankProfilesConfigImporterWithMockedConstants(Path constantsPath, FileAcquirer fileAcquirer) { + super(fileAcquirer); this.constantsPath = constantsPath; } @Override - protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + protected Tensor readTensorFromFile(String name, TensorType type, FileReference fileReference) { try { return TypedBinaryFormat.decode(Optional.of(type), GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantsPath.append(name).toFile()))); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index 0708d872ff6..bd1ff6b8ed7 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -3,6 +3,7 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.FileSource; +import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; import com.yahoo.vespa.config.search.RankProfilesConfig; @@ -47,7 +48,7 @@ public class ModelsEvaluatorTest { RankProfilesConfig.class).getConfig(""); RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), RankingConstantsConfig.class).getConfig(""); - return new ModelsEvaluator(config, constantsConfig); + return new ModelsEvaluator(config, constantsConfig, MockFileAcquirer.returnFile(null)); } } |