diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-11-26 15:00:16 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@oath.com> | 2018-11-26 15:00:16 +0100 |
commit | e3c64c48445d85054292b6fe90c3347d5ec327f2 (patch) | |
tree | 9c4f8509699a74ec614633159d1eb3171482d456 /config-model-api | |
parent | 04cc3c48130b8397c04335948e5971914b2eaf22 (diff) |
Model the config model view of model-integration
This is to make it clearer that these methods are part
of the config model API.
Diffstat (limited to 'config-model-api')
4 files changed, 181 insertions, 0 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ImportedMlFunction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ImportedMlFunction.java new file mode 100644 index 00000000000..54cdf807878 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ImportedMlFunction.java @@ -0,0 +1,37 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * An imported function of an imported machine-learned model + * + * @author bratseth + */ +public class ImportedMlFunction { + + private final String name; + private final List<String> arguments; + private final Map<String, String> argumentTypes; + private final String expression; + private final Optional<String> returnType; + + public ImportedMlFunction(String name, List<String> arguments, String expression, + Map<String, String> argumentTypes, Optional<String> returnType) { + this.name = name; + this.arguments = Collections.unmodifiableList(arguments); + this.expression = expression; + this.argumentTypes = Collections.unmodifiableMap(argumentTypes); + this.returnType = returnType; + } + + public String name() { return name; } + public List<String> arguments() { return arguments; } + public Map<String, String> argumentTypes() { return argumentTypes; } + public String expression() { return expression; } + public Optional<String> returnType() { return returnType; } + +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ImportedMlModel.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ImportedMlModel.java new file mode 100644 index 00000000000..078e4c239d6 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ImportedMlModel.java @@ -0,0 +1,23 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Config model view of an imported machine-learned model. + * + * @author bratseth + */ +public interface ImportedMlModel { + + String name(); + String source(); + Optional<String> inputTypeSpec(String input); + Map<String, String> smallConstants(); + Map<String, String> largeConstants(); + Map<String, String> functions(); + List<ImportedMlFunction> outputExpressions(); + +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ImportedMlModels.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ImportedMlModels.java new file mode 100644 index 00000000000..aeef81788b8 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ImportedMlModels.java @@ -0,0 +1,105 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +import com.yahoo.path.Path; + +import java.io.File; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * All models imported from the models/ directory in the application package. + * If this is empty it may be due to either not having any models in the application package, + * or this being created for a ZooKeeper application package, which does not have imported models. + * + * @author bratseth + */ +public class ImportedMlModels { + + /** All imported models, indexed by their names */ + private final Map<String, ImportedMlModel> importedModels; + + /** Create a null imported models */ + public ImportedMlModels() { + importedModels = Collections.emptyMap(); + } + + public ImportedMlModels(File modelsDirectory, Collection<MlModelImporter> importers) { + Map<String, ImportedMlModel> models = new HashMap<>(); + + // Find all subdirectories recursively which contains a model we can read + importRecursively(modelsDirectory, models, importers); + importedModels = Collections.unmodifiableMap(models); + } + + /** + * Returns the model at the given location in the application package. + * + * @param modelPath the path to this model (file or directory, depending on model type) + * under the application package, both from the root or relative to the + * models directory works + * @return the model at this path or null if none + */ + public ImportedMlModel get(File modelPath) { + return importedModels.get(toName(modelPath)); + } + + /** Returns an immutable collection of all the imported models */ + public Collection<ImportedMlModel> all() { + return importedModels.values(); + } + + private static void importRecursively(File dir, + Map<String, ImportedMlModel> models, + Collection<MlModelImporter> importers) { + if ( ! dir.isDirectory()) return; + + Arrays.stream(dir.listFiles()).sorted().forEach(child -> { + Optional<MlModelImporter> importer = findImporterOf(child, importers); + if (importer.isPresent()) { + String name = toName(child); + ImportedMlModel existing = models.get(name); + if (existing != null) + throw new IllegalArgumentException("The models in " + child + " and " + existing.source() + + " both resolve to the model name '" + name + "'"); + models.put(name, importer.get().importModel(name, child)); + } + else { + importRecursively(child, models, importers); + } + }); + } + + private static Optional<MlModelImporter> findImporterOf(File path, Collection<MlModelImporter> importers) { + return importers.stream().filter(item -> item.canImport(path.toString())).findFirst(); + } + + private static String toName(File modelFile) { + Path modelPath = Path.fromString(modelFile.toString()); + if (modelFile.isFile()) + modelPath = stripFileEnding(modelPath); + String localPath = concatenateAfterModelsDirectory(modelPath); + return localPath.replace('.', '_'); + } + + private static Path stripFileEnding(Path path) { + int dotIndex = path.last().lastIndexOf("."); + if (dotIndex <= 0) return path; + return path.withLast(path.last().substring(0, dotIndex)); + } + + private static String concatenateAfterModelsDirectory(Path path) { + boolean afterModels = false; + StringBuilder result = new StringBuilder(); + for (String element : path.elements()) { + if (afterModels) result.append(element).append("_"); + if (element.equals("models")) afterModels = true; + } + return result.substring(0, result.length()-1); + } + +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/MlModelImporter.java b/config-model-api/src/main/java/com/yahoo/config/model/api/MlModelImporter.java new file mode 100644 index 00000000000..d24eeb2d55a --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/MlModelImporter.java @@ -0,0 +1,16 @@ +package com.yahoo.config.model.api; + +import java.io.File; + +/** + * Config model view of a machine-learned model importer + * + * @author bratseth + */ +public interface MlModelImporter { + + boolean canImport(String modelPath); + + ImportedMlModel importModel(String modelName, File modelPath); + +} |