diff options
36 files changed, 616 insertions, 218 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java index 13fb57dc1b8..e0976c0eaef 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -15,9 +15,8 @@ import java.util.List; import java.util.logging.Logger; /** - * @author lulf - * @author vegardh - * @since 5.1 + * @author Ulf Lilleengen + * @author Vegard Havdal */ public class FilesApplicationFile extends ApplicationFile { @@ -198,4 +197,5 @@ public class FilesApplicationFile extends ApplicationFile { if (other == this) return 0; return this.getPath().getName().compareTo((other).getPath().getName()); } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 5367bdcf13f..13d9283d151 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -55,7 +55,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; * Construct using {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFile(java.io.File)} or * {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFileWithDeployData(java.io.File, DeployData)}. * - * @author vegardh + * @author Vegard Havdal */ public class FilesApplicationPackage implements ApplicationPackage { @@ -97,15 +97,15 @@ public class FilesApplicationPackage implements ApplicationPackage { } /** Creates package from a local directory, typically deploy app */ - public static FilesApplicationPackage fromFileWithDeployData(File appDir, DeployData deployData, + public static FilesApplicationPackage fromFileWithDeployData(File appDir, DeployData deployData, boolean includeSourceFiles) { return new Builder(appDir).includeSourceFiles(includeSourceFiles).deployData(deployData).build(); } private static ApplicationMetaData metaDataFromDeployData(File appDir, DeployData deployData) { - return new ApplicationMetaData(deployData.getDeployedByUser(), deployData.getDeployedFromDir(), - deployData.getDeployTimestamp(), deployData.getApplicationName(), - computeCheckSum(appDir), deployData.getGeneration(), + return new ApplicationMetaData(deployData.getDeployedByUser(), deployData.getDeployedFromDir(), + deployData.getDeployTimestamp(), deployData.getApplicationName(), + computeCheckSum(appDir), deployData.getGeneration(), deployData.getCurrentlyActiveGeneration()); } @@ -385,9 +385,9 @@ public class FilesApplicationPackage implements ApplicationPackage { } } - /** + /** * Creates a reader for a config definition - * + * * @param defPath the path to the application package * @return the reader of this config definition */ @@ -456,10 +456,10 @@ public class FilesApplicationPackage implements ApplicationPackage { if (defs.containsKey(key)) { if (nv[0].contains(".")) { - log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + + log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + ". The file '" + def + "' will take precedence"); } else { - log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + + log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + ". Skipping '" + def + "', as it does not contain namespace in filename"); continue; // skip } @@ -709,7 +709,7 @@ public class FilesApplicationPackage implements ApplicationPackage { } } - + /** * Adds the given path to the digest, or does nothing if path is neither file nor dir * @param path path to add to message digest diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index acb537debe7..6da2b0f8125 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -53,7 +53,13 @@ public interface ApplicationPackage { String DOCPROCCHAINS_DIR = "docproc/chains"; String PROCESSORCHAINS_DIR = "processor/chains"; String ROUTINGTABLES_DIR = "routing/tables"; - String MODELS_DIR = "models"; + + /** Machine-learned models - only present in user-uploaded package instances */ + Path MODELS_DIR = Path.fromString("models"); + /** Files generated from machine-learned models */ + Path MODELS_GENERATED_DIR = Path.fromString("models.generated"); + /** Files generated from machine-learned models which should be replicated in ZooKeeper */ + Path MODELS_GENERATED_REPLICATED_DIR = MODELS_GENERATED_DIR.append("replicated"); // NOTE: this directory is created in serverdb during deploy, and should not exist in the original user application /** Do not use */ @@ -131,7 +137,7 @@ public interface ApplicationPackage { */ List<NamedReader> getFiles(Path pathFromRoot, String suffix, boolean recurse); - /** Same as getFiles(pathFromRoot,suffix,false) */ + /** Same as getFiles(pathFromRoot, suffix, false) */ default List<NamedReader> getFiles(Path pathFromRoot, String suffix) { return getFiles(pathFromRoot,suffix,false); } diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java index 5daf5ca70a5..385cd883da4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java @@ -8,7 +8,7 @@ package com.yahoo.config.model; * * @author gjoranv * @author bratseth - * @author lulf + * @author Ulf Lilleengen */ public abstract class ConfigModel { diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index 5eb4afcc241..5912b476783 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -21,8 +21,7 @@ import java.util.*; /** * Config model adaptor of the Admin class. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class AdminModel extends ConfigModel { @@ -46,8 +45,9 @@ public class AdminModel extends ConfigModel { @Override public void prepare(ConfigModelRepo configModelRepo) { verifyClusterControllersOnlyDefinedForContent(configModelRepo); - if (admin == null || admin.getClusterControllers() == null) return; - admin.getClusterControllers().prepare(); + if (admin == null) return; + if (admin.getClusterControllers() != null) + admin.getClusterControllers().prepare(); } private void verifyClusterControllersOnlyDefinedForContent(ConfigModelRepo configModelRepo) { @@ -61,9 +61,9 @@ public class AdminModel extends ConfigModel { public static class BuilderV2 extends ConfigModelBuilder<AdminModel> { public static final List<ConfigModelId> configModelIds = - ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "2.0"), + ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "2.0"), ConfigModelId.fromNameAndVersion("admin", "1.0")); - + public BuilderV2() { super(AdminModel.class); } @@ -91,7 +91,7 @@ public class AdminModel extends ConfigModel { public static class BuilderV4 extends ConfigModelBuilder<AdminModel> { public static final List<ConfigModelId> configModelIds = - ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "3.0"), + ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "3.0"), ConfigModelId.fromNameAndVersion("admin", "4.0")); public BuilderV4() { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index ddee0be6e9c..271ec6958ec 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -37,8 +37,8 @@ public class MockApplicationPackage implements ApplicationPackage { private final Optional<String> validationOverrides; private final boolean failOnValidateXml; - private MockApplicationPackage(String hosts, String services, List<String> searchDefinitions, String searchDefinitionDir, - String deploymentSpec, String validationOverrides, boolean failOnValidateXml) { + protected MockApplicationPackage(String hosts, String services, List<String> searchDefinitions, String searchDefinitionDir, + String deploymentSpec, String validationOverrides, boolean failOnValidateXml) { this.hostsS = hosts; this.servicesS = services; this.searchDefinitions = searchDefinitions; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index f37ab9fb89f..df5697de0d5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -165,9 +165,8 @@ public class Search implements Serializable, ImmutableSearch { public void addRankingConstant(RankingConstant constant) { constant.validate(); String name = constant.getName(); - if (rankingConstants.get(name) != null) { - throw new IllegalArgumentException("Ranking constant '"+name+"' defined twice"); - } + if (rankingConstants.containsKey(name)) + throw new IllegalArgumentException("Ranking constant '" + name + "' defined twice"); rankingConstants.put(name, constant); } @@ -268,6 +267,8 @@ public class Search implements Serializable, ImmutableSearch { return sourceApplication.getRankingExpression(fileName); } + public ApplicationPackage sourceApplication() { return sourceApplication; } + /** * Returns a field defined in this search definition or one if its documents. Fields in this search definition takes * precedence over document fields having the same name diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java index 32f8f4871df..0dd5b4166ef 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java @@ -2,14 +2,17 @@ package com.yahoo.searchdefinition.expressiontransforms; import com.google.common.base.Joiner; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowImporter; import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowModel; +import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowModel.Signature; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.Arguments; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; @@ -17,12 +20,16 @@ import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; import java.io.File; import java.io.IOException; +import java.io.StringReader; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -33,17 +40,14 @@ import java.util.Optional; * * @author bratseth */ -// TODO: - Verify types of macros -// - Avoid name conflicts across models for constants +// TODO: Verify types of macros +// TODO: Avoid name conflicts across models for constants public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> { - // TODO: Make system test work with this set to true, then remove the "true" path - private static final boolean constantsInConfig = true; - private final TensorFlowImporter tensorFlowImporter = new TensorFlowImporter(); /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ - private final Map<String, TensorFlowModel> importedModels = new HashMap<>(); + private final Map<Path, TensorFlowModel> importedModels = new HashMap<>(); @Override public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { @@ -56,40 +60,48 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil } private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { - try { - if ( ! feature.getName().equals("tensorflow")) return feature; + if ( ! feature.getName().equals("tensorflow")) return feature; - if (feature.getArguments().isEmpty()) - throw new IllegalArgumentException("A tensorflow node must take an argument pointing to " + - "the tensorflow model directory under [application]/models"); + try { + ModelStore store = new ModelStore(context.rankProfile().getSearch().sourceApplication(), + feature.getArguments()); + if (store.hasTensorFlowModels()) + return transformFromTensorFlowModel(store, context.rankProfile()); + else // is should have previously stored model information instead + return transformFromStoredModel(store, context.rankProfile()); + } + catch (IllegalArgumentException | UncheckedIOException e) { + throw new IllegalArgumentException("Could not use tensorflow model from " + feature, e); + } + } - String modelPath = ApplicationPackage.MODELS_DIR + "/" + asString(feature.getArguments().expressions().get(0)); - TensorFlowModel result = importedModels.computeIfAbsent(modelPath, k -> tensorFlowImporter.importModel(modelPath)); + private ExpressionNode transformFromTensorFlowModel(ModelStore store, RankProfile profile) { + TensorFlowModel model = importedModels.computeIfAbsent(store.arguments().modelPath(), + k -> tensorFlowImporter.importModel(store.tensorFlowModelDir())); - // Find the specified expression - TensorFlowModel.Signature signature = chooseSignature(result, - optionalArgument(1, feature.getArguments())); - RankingExpression expression = chooseOutput(signature, - optionalArgument(2, feature.getArguments())); + // Find the specified expression + Signature signature = chooseSignature(model, store.arguments().signature()); + String output = chooseOutput(signature, store.arguments().output()); + RankingExpression expression = model.expressions().get(output); + store.writeConverted(expression); - // Add all constants (after finding outputs to fail faster when the output is not found) - if (constantsInConfig) - result.constants().forEach((k, v) -> context.rankProfile().addConstantTensor(k, new TensorValue(v))); - else // correct way, disabled for now - result.constants().forEach((k, v) -> transformConstant(modelPath, context.rankProfile(), k, v)); + model.constants().forEach((k, v) -> transformConstant(store, profile, k, v)); + return expression.getRoot(); + } - return expression.getRoot(); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not use tensorflow model from " + feature, e); + private ExpressionNode transformFromStoredModel(ModelStore store, RankProfile profile) { + for (RankingConstant constant : store.readRankingConstants()) { + if (!profile.getSearch().getRankingConstants().containsKey(constant.getName())) + profile.getSearch().addRankingConstant(constant); } + return store.readConverted().getRoot(); } /** * Returns the specified, existing signature, or the only signature if none is specified. * Throws IllegalArgumentException in all other cases. */ - private TensorFlowModel.Signature chooseSignature(TensorFlowModel importResult, Optional<String> signatureName) { + private Signature chooseSignature(TensorFlowModel importResult, Optional<String> signatureName) { if ( ! signatureName.isPresent()) { if (importResult.signatures().size() == 0) throw new IllegalArgumentException("No signatures are available"); @@ -101,7 +113,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil return importResult.signatures().values().stream().findFirst().get(); } else { - TensorFlowModel.Signature signature = importResult.signatures().get(signatureName.get()); + Signature signature = importResult.signatures().get(signatureName.get()); if (signature == null) throw new IllegalArgumentException("Model does not have the specified signature '" + signatureName.get() + "'"); @@ -113,7 +125,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil * Returns the specified, existing output expression, or the only output expression if no output name is specified. * Throws IllegalArgumentException in all other cases. */ - private RankingExpression chooseOutput(TensorFlowModel.Signature signature, Optional<String> outputName) { + private String chooseOutput(Signature signature, Optional<String> outputName) { if ( ! outputName.isPresent()) { if (signature.outputs().size() == 0) throw new IllegalArgumentException("No outputs are available" + skippedOutputsDescription(signature)); @@ -122,11 +134,11 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil Joiner.on(", ").join(signature.outputs().keySet()) + "), one must be specified " + "as a third argument to tensorflow()"); - return signature.outputExpression(signature.outputs().keySet().stream().findFirst().get()); + return signature.outputs().get(signature.outputs().keySet().stream().findFirst().get()); } else { - RankingExpression expression = signature.outputExpression(outputName.get()); - if (expression == null) { + String output = signature.outputs().get(outputName.get()); + if (output == null) { if (signature.skippedOutputs().containsKey(outputName.get())) throw new IllegalArgumentException("Could not use output '" + outputName.get() + "': " + signature.skippedOutputs().get(outputName.get())); @@ -134,28 +146,16 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil throw new IllegalArgumentException("Model does not have the specified output '" + outputName.get() + "'"); } - return expression; + return output; } } - private void transformConstant(String modelPath, RankProfile profile, String constantName, Tensor constantValue) { - try { - if (profile.getSearch().getRankingConstants().containsKey(constantName)) return; + private void transformConstant(ModelStore store, RankProfile profile, String constantName, Tensor constantValue) { + if (profile.getSearch().getRankingConstants().containsKey(constantName)) return; - File constantFilePath = new File(modelPath, "converted_variables").getCanonicalFile(); - if (!constantFilePath.exists()) { - if (!constantFilePath.mkdir()) - throw new IOException("Could not create directory " + constantFilePath); - } - - // "tbf" ending for "typed binary format" - recognized by the nodes reciving the file: - File constantFile = new File(constantFilePath, constantName + ".tbf"); - IOUtils.writeFile(constantFile, TypedBinaryFormat.encode(constantValue)); - profile.getSearch().addRankingConstant(new RankingConstant(constantName, constantValue.type(), constantFile.getPath())); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } + Path constantPath = store.writeConstant(constantName, constantValue); + profile.getSearch().addRankingConstant(new RankingConstant(constantName, constantValue.type(), + constantPath.toString())); } private String skippedOutputsDescription(TensorFlowModel.Signature signature) { @@ -165,27 +165,176 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil return b.toString(); } - private Optional<String> optionalArgument(int argumentIndex, Arguments arguments) { - if (argumentIndex >= arguments.expressions().size()) - return Optional.empty(); - return Optional.of(asString(arguments.expressions().get(argumentIndex))); - } + /** + * Provides read/write access to the correct directories of the application package given by the feature arguments + */ + private static class ModelStore { - private String asString(ExpressionNode node) { - if ( ! (node instanceof ConstantNode)) - throw new IllegalArgumentException("Expected a constant string as tensorflow argument, but got '" + node); - return stripQuotes(((ConstantNode)node).sourceString()); - } + private final ApplicationPackage application; + private final FeatureArguments arguments; + + public ModelStore(ApplicationPackage application, Arguments arguments) { + this.application = application; + this.arguments = new FeatureArguments(arguments); + } + + public FeatureArguments arguments() { return arguments; } + + public boolean hasTensorFlowModels() { + try { + return application.getFileReference(ApplicationPackage.MODELS_DIR).exists(); + } + catch (UnsupportedOperationException e) { + return false; // No files -> no TensorFlow models + } + } + + /** + * Returns the directory which (if hasTensorFlowModels is true) + * contains the source model to use for these arguments + */ + public File tensorFlowModelDir() { + return application.getFileReference(ApplicationPackage.MODELS_DIR.append(arguments.modelPath())); + } + + /** + * Adds this expression to the application package, such that it can be read later. + */ + public void writeConverted(RankingExpression expression) { + application.getFile(arguments.expressionPath()) + .writeFile(new StringReader(expression.getRoot().toString())); + } + + /** Reads the previously stored ranking expression for these arguments */ + public RankingExpression readConverted() { + try { + return new RankingExpression(application.getFile(arguments.expressionPath()).createReader()); + } + catch (IOException e) { + throw new UncheckedIOException("Could not read " + arguments.expressionPath(), e); + } + catch (ParseException e) { + throw new IllegalStateException("Could not parse " + arguments.expressionPath(), e); + } + } + + /** + * Reads the information about all the constants stored in the application package + * (the constant value itself is replicated with file distribution). + */ + public List<RankingConstant> readRankingConstants() { + try { + List<RankingConstant> constants = new ArrayList<>(); + for (ApplicationFile constantFile : application.getFile(arguments.rankingConstantsPath()).listFiles()) { + String[] parts = IOUtils.readAll(constantFile.createReader()).split(":"); + constants.add(new RankingConstant(parts[0], TensorType.fromSpec(parts[1]), parts[2])); + } + return constants; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Adds this constant to the application package as a file, + * such that it can be distributed using file distribution. + * + * @return the path to the stored constant, relative to the application package root + */ + public Path writeConstant(String name, Tensor constant) { + Path constantsPath = ApplicationPackage.MODELS_GENERATED_DIR.append(arguments.modelPath).append("constants"); + + // "tbf" ending for "typed binary format" - recognized by the nodes receiving the file: + Path constantPath = constantsPath.append(name + ".tbf"); + + // Remember the constant in a file we replicate in ZooKeeper + application.getFile(arguments.rankingConstantsPath().append(name + ".constant")) + .writeFile(new StringReader(name + ":" + constant.type() + ":" + constantPath)); + + // Write content explicitly as a file on the file system as this is distributed using file distribution + createIfNeeded(constantsPath); + IOUtils.writeFile(application.getFileReference(constantPath), TypedBinaryFormat.encode(constant)); + return constantPath; + } + + private void createIfNeeded(Path path) { + File dir = application.getFileReference(path); + if ( ! dir.exists()) { + if (!dir.mkdirs()) + throw new IllegalStateException("Could not create " + dir); + } + } - private String stripQuotes(String s) { - if ( ! isQuoteSign(s.codePointAt(0))) return s; - if ( ! isQuoteSign(s.codePointAt(s.length() - 1 ))) - throw new IllegalArgumentException("tensorflow argument [" + s + "] is missing endquote"); - return s.substring(1, s.length()-1); } - private boolean isQuoteSign(int c) { - return c == '\'' || c == '"'; + /** Encapsulates the 1, 2 or 3 arguments to a tensorflow feature */ + private static class FeatureArguments { + + private final Path modelPath; + + /** Optional arguments */ + private final Optional<String> signature, output; + + public FeatureArguments(Arguments arguments) { + if (arguments.isEmpty()) + throw new IllegalArgumentException("A tensorflow node must take an argument pointing to " + + "the tensorflow model directory under [application]/models"); + if (arguments.expressions().size() > 3) + throw new IllegalArgumentException("A tensorflow feature can have at most 3 arguments"); + + modelPath = Path.fromString(asString(arguments.expressions().get(0))); + signature = optionalArgument(1, arguments); + output = optionalArgument(2, arguments); + } + + /** Returns relative path to this model below the "models/" dir in the application package */ + public Path modelPath() { return modelPath; } + public Optional<String> signature() { return signature; } + public Optional<String> output() { return output; } + + public Path rankingConstantsPath() { + return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(modelPath).append("constants"); + } + + public Path expressionPath() { + return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR + .append(modelPath).append("expressions").append(expressionFileName()); + } + + private String expressionFileName() { + StringBuilder fileName = new StringBuilder(); + signature.ifPresent(s -> fileName.append(s).append(".")); + output.ifPresent(s -> fileName.append(s).append(".")); + if (fileName.length() == 0) // single signature and output + fileName.append("single."); + fileName.append("expression"); + return fileName.toString(); + } + + private Optional<String> optionalArgument(int argumentIndex, Arguments arguments) { + if (argumentIndex >= arguments.expressions().size()) + return Optional.empty(); + return Optional.of(asString(arguments.expressions().get(argumentIndex))); + } + + private String asString(ExpressionNode node) { + if ( ! (node instanceof ConstantNode)) + throw new IllegalArgumentException("Expected a constant string as tensorflow argument, but got '" + node); + return stripQuotes(((ConstantNode)node).sourceString()); + } + + private String stripQuotes(String s) { + if ( ! isQuoteSign(s.codePointAt(0))) return s; + if ( ! isQuoteSign(s.codePointAt(s.length() - 1 ))) + throw new IllegalArgumentException("tensorflow argument [" + s + "] is missing endquote"); + return s.substring(1, s.length()-1); + } + + private boolean isQuoteSign(int c) { + return c == '\'' || c == '"'; + } + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 59b7388f5bb..071b3090f99 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -73,9 +73,7 @@ public class Admin extends AbstractConfigProducer implements Serializable { this.fileDistribution = fileDistributionConfigProducer; } - public Configserver getConfigserver() { - return defaultConfigserver; - } + public Configserver getConfigserver() { return defaultConfigserver; } /** Returns the configured monitoring endpoint, or null if not configured */ public Monitoring getMonitoring() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index fd062dc4ea4..58fc76f1508 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -49,16 +49,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer public static final IndexingMode REALTIME = new IndexingMode("REALTIME"); public static final IndexingMode STREAMING = new IndexingMode("STREAMING"); - public static IndexingMode createIndexingMode(String ixm) { - if ("REALTIME".equalsIgnoreCase(ixm)) { - return REALTIME; - } else if ("STREAMING".equalsIgnoreCase(ixm)) { - return STREAMING; - } - return null; - } - - private String name; + private final String name; private IndexingMode(String name) { this.name = name; @@ -72,6 +63,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer } public static final class SearchDefinitionSpec { + private final SearchDefinition searchDefinition; private final UserConfigRepo userConfigRepo; diff --git a/config-model/src/test/integration/tensorflow/mnist_softmax/mnist_sftmax_with_saving.py b/config-model/src/test/integration/tensorflow/models/mnist_softmax/mnist_sftmax_with_saving.py index a1861a1c981..a1861a1c981 100644 --- a/config-model/src/test/integration/tensorflow/mnist_softmax/mnist_sftmax_with_saving.py +++ b/config-model/src/test/integration/tensorflow/models/mnist_softmax/mnist_sftmax_with_saving.py diff --git a/config-model/src/test/integration/tensorflow/mnist_softmax/saved/saved_model.pbtxt b/config-model/src/test/integration/tensorflow/models/mnist_softmax/saved/saved_model.pbtxt index 8100dfd594d..8100dfd594d 100644 --- a/config-model/src/test/integration/tensorflow/mnist_softmax/saved/saved_model.pbtxt +++ b/config-model/src/test/integration/tensorflow/models/mnist_softmax/saved/saved_model.pbtxt diff --git a/config-model/src/test/integration/tensorflow/mnist_softmax/saved/variables/variables.data-00000-of-00001 b/config-model/src/test/integration/tensorflow/models/mnist_softmax/saved/variables/variables.data-00000-of-00001 Binary files differindex 8474aa0a04c..8474aa0a04c 100644 --- a/config-model/src/test/integration/tensorflow/mnist_softmax/saved/variables/variables.data-00000-of-00001 +++ b/config-model/src/test/integration/tensorflow/models/mnist_softmax/saved/variables/variables.data-00000-of-00001 diff --git a/config-model/src/test/integration/tensorflow/mnist_softmax/saved/variables/variables.index b/config-model/src/test/integration/tensorflow/models/mnist_softmax/saved/variables/variables.index Binary files differindex cfcdac20409..cfcdac20409 100644 --- a/config-model/src/test/integration/tensorflow/mnist_softmax/saved/variables/variables.index +++ b/config-model/src/test/integration/tensorflow/models/mnist_softmax/saved/variables/variables.index diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index ff53fdafacf..7c749608e1f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -1,5 +1,7 @@ package com.yahoo.searchdefinition.processing; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; @@ -22,7 +24,11 @@ class RankProfileSearchFixture { private Search search; RankProfileSearchFixture(String rankProfiles) throws ParseException { - SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + this(MockApplicationPackage.createEmpty(), rankProfiles); + } + + RankProfileSearchFixture(ApplicationPackage applicationpackage, String rankProfiles) throws ParseException { + SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry); String sdContent = "search test {\n" + " document test {\n" + " }\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 31f7511155b..0354173f365 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -1,24 +1,36 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.yolean.Exceptions; import org.junit.After; import org.junit.Test; +import java.io.BufferedInputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -27,47 +39,52 @@ import static org.junit.Assert.fail; */ public class RankingExpressionWithTensorFlowTestCase { - // The "../" is to escape the "models/" element prepended to the path - private final String modelDirectory = "../src/test/integration/tensorflow/mnist_softmax/saved"; + private final Path applicationDir = Path.fromString("src/test/integration/tensorflow/"); private final String vespaExpression = "join(rename(reduce(join(Placeholder, rename(constant(Variable), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(Variable_1), d0, d1), f(a,b)(a + b))"; @After public void removeGeneratedConstantTensorFiles() { - IOUtils.recursiveDeleteDir(new File(modelDirectory.substring(3), "converted_variables")); + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); } @Test public void testMinimalTensorFlowReference() throws ParseException { + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); RankProfileSearchFixture search = new RankProfileSearchFixture( + application, " rank-profile my_profile {\n" + " first-phase {\n" + - " expression: tensorflow('" + modelDirectory + "')" + + " expression: tensorflow('mnist_softmax/saved')" + " }\n" + " }"); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - assertConstant(10, "Variable_1", search); - assertConstant(7840, "Variable", search); + assertConstant("Variable_1", search, Optional.of(10L)); + assertConstant("Variable", search, Optional.of(7840L)); } @Test public void testNestedTensorFlowReference() throws ParseException { + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); RankProfileSearchFixture search = new RankProfileSearchFixture( + application, " rank-profile my_profile {\n" + " first-phase {\n" + - " expression: 5 + sum(tensorflow('" + modelDirectory + "'))" + + " expression: 5 + sum(tensorflow('mnist_softmax/saved'))" + " }\n" + " }"); search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile"); - assertConstant(10, "Variable_1", search); - assertConstant(7840, "Variable", search); + assertConstant("Variable_1", search, Optional.of(10L)); + assertConstant("Variable", search, Optional.of(7840L)); } @Test public void testTensorFlowReferenceSpecifyingSignature() throws ParseException { + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); RankProfileSearchFixture search = new RankProfileSearchFixture( + application, " rank-profile my_profile {\n" + " first-phase {\n" + - " expression: tensorflow('" + modelDirectory + "', 'serving_default')" + + " expression: tensorflow('mnist_softmax/saved', 'serving_default')" + " }\n" + " }"); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); @@ -75,10 +92,12 @@ public class RankingExpressionWithTensorFlowTestCase { @Test public void testTensorFlowReferenceSpecifyingSignatureAndOutput() throws ParseException { + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); RankProfileSearchFixture search = new RankProfileSearchFixture( + application, " rank-profile my_profile {\n" + " first-phase {\n" + - " expression: tensorflow('" + modelDirectory + "', 'serving_default', 'y')" + + " expression: tensorflow('mnist_softmax/saved', 'serving_default', 'y')" + " }\n" + " }"); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); @@ -87,18 +106,21 @@ public class RankingExpressionWithTensorFlowTestCase { @Test public void testTensorFlowReferenceSpecifyingNonExistingSignature() throws ParseException { try { + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); RankProfileSearchFixture search = new RankProfileSearchFixture( + application, " rank-profile my_profile {\n" + " first-phase {\n" + - " expression: tensorflow('" + modelDirectory + "', 'serving_defaultz')" + + " expression: tensorflow('mnist_softmax/saved', 'serving_defaultz')" + " }\n" + " }"); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); fail("Expecting exception"); } catch (IllegalArgumentException expected) { - assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from tensorflow('" + - modelDirectory + "','serving_defaultz'): Model does not have the specified signature 'serving_defaultz'", + assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + + "tensorflow('mnist_softmax/saved','serving_defaultz'): " + + "Model does not have the specified signature 'serving_defaultz'", Exceptions.toMessageString(expected)); } } @@ -106,36 +128,83 @@ public class RankingExpressionWithTensorFlowTestCase { @Test public void testTensorFlowReferenceSpecifyingNonExistingOutput() throws ParseException { try { + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); RankProfileSearchFixture search = new RankProfileSearchFixture( + application, " rank-profile my_profile {\n" + " first-phase {\n" + - " expression: tensorflow('" + modelDirectory + "', 'serving_default', 'x')" + + " expression: tensorflow('mnist_softmax/saved', 'serving_default', 'x')" + " }\n" + " }"); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); fail("Expecting exception"); } catch (IllegalArgumentException expected) { - assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from tensorflow('" + - modelDirectory + "','serving_default','x'): Model does not have the specified output 'x'", + assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + + "tensorflow('mnist_softmax/saved','serving_default','x'): " + + "Model does not have the specified output 'x'", Exceptions.toMessageString(expected)); } } - private void assertConstant(int expectedSize, String name, RankProfileSearchFixture search) { + @Test + public void testImportingFromStoredExpressions() throws ParseException, IOException { + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); + RankProfileSearchFixture search = new RankProfileSearchFixture( + application, + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: tensorflow('mnist_softmax/saved', 'serving_default')" + + " }\n" + + " }"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + assertConstant("Variable_1", search, Optional.of(10L)); + assertConstant("Variable", search, Optional.of(7840L)); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = new RankProfileSearchFixture( + storedApplication, + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: tensorflow('mnist_softmax/saved', 'serving_default')" + + " }\n" + + " }"); + searchFromStored.assertFirstPhaseExpression(vespaExpression, "my_profile"); + // Verify that the constants exists, but don't verify the content as we are not + // simulating file distribution in this test + assertConstant("Variable_1", searchFromStored, Optional.empty()); + assertConstant("Variable", searchFromStored, Optional.empty()); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + + } + + /** + * Verifies that the constant with the given name exists, and - only if an expected size is given - + * that the content of the constant is available and has the expected size. + */ + private void assertConstant(String name, RankProfileSearchFixture search, Optional<Long> expectedSize) { try { - TensorValue constant = (TensorValue)search.rankProfile("my_profile").getConstants().get(name); // Old way. TODO: Remove - if (constant == null) { // New way - File constantFile = new File(modelDirectory.substring(3) + "/converted_variables", name + ".tbf"); - RankingConstant rankingConstant = search.search().getRankingConstants().get(name); - assertEquals(name, rankingConstant.getName()); - assertEquals(constantFile.getAbsolutePath(), rankingConstant.getFileName()); - assertTrue("Constant file has been written", constantFile.exists()); - Tensor deserializedConstant = TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantFile))); - assertEquals(expectedSize, deserializedConstant.size()); - } else { // Old way. TODO: Remove - assertNotNull(name + " is imported", constant); - assertEquals(expectedSize, constant.asTensor().size()); + Path constantApplicationPackagePath = Path.fromString("models.generated/mnist_softmax/saved/constants").append(name + ".tbf"); + RankingConstant rankingConstant = search.search().getRankingConstants().get(name); + assertEquals(name, rankingConstant.getName()); + assertEquals(constantApplicationPackagePath.toString(), rankingConstant.getFileName()); + + if (expectedSize.isPresent()) { + Path constantPath = applicationDir.append(constantApplicationPackagePath); + assertTrue("Constant file '" + constantPath + "' has been written", + constantPath.toFile().exists()); + Tensor deserializedConstant = TypedBinaryFormat.decode(Optional.empty(), + GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantPath.toFile()))); + assertEquals(expectedSize.get().longValue(), deserializedConstant.size()); } } catch (IOException e) { @@ -143,4 +212,138 @@ public class RankingExpressionWithTensorFlowTestCase { } } + private static class StoringApplicationPackage extends MockApplicationPackage { + + private final File root; + + StoringApplicationPackage(Path applicationPackageWritableRoot) { + this(applicationPackageWritableRoot.toFile()); + } + + StoringApplicationPackage(File applicationPackageWritableRoot) { + super(null, null, Collections.emptyList(), null, + null, null, false); + this.root = applicationPackageWritableRoot; + } + + @Override + public File getFileReference(Path path) { + return Path.fromString(root.toString()).append(path).toFile(); + } + + @Override + public ApplicationFile getFile(Path file) { + return new StoringApplicationPackageFile(file, Path.fromString(root.toString())); + } + + } + + private static class StoringApplicationPackageFile extends ApplicationFile { + + /** The path to the application package root */ + private final Path root; + + /** The File pointing to the actual file represented by this */ + private final File file; + + StoringApplicationPackageFile(Path filePath, Path applicationPackagePath) { + super(filePath); + this.root = applicationPackagePath; + file = applicationPackagePath.append(filePath).toFile(); + } + + @Override + public boolean isDirectory() { + return file.isDirectory(); + } + + @Override + public boolean exists() { + return file.exists(); + } + + @Override + public Reader createReader() throws FileNotFoundException { + try { + if ( ! exists()) throw new FileNotFoundException("File '" + file + "' does not exist"); + return IOUtils.createReader(file, "UTF-8"); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public InputStream createInputStream() throws FileNotFoundException { + try { + if ( ! exists()) throw new FileNotFoundException("File '" + file + "' does not exist"); + return new BufferedInputStream(new FileInputStream(file)); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public ApplicationFile createDirectory() { + file.mkdirs(); + return this; + } + + @Override + public ApplicationFile writeFile(Reader input) { + try { + IOUtils.writeFile(file, IOUtils.readAll(input), false); + return this; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List<ApplicationFile> listFiles(PathFilter filter) { + if ( ! isDirectory()) return Collections.emptyList(); + return Arrays.stream(file.listFiles()).filter(f -> filter.accept(Path.fromString(f.toString()))) + .map(f -> new StoringApplicationPackageFile(asApplicationRelativePath(f), + root)) + .collect(Collectors.toList()); + } + + @Override + public ApplicationFile delete() { + file.delete(); + return this; + } + + @Override + public MetaData getMetaData() { + throw new UnsupportedOperationException(); + } + + @Override + public int compareTo(ApplicationFile other) { + return this.getPath().getName().compareTo((other).getPath().getName()); + } + + /** Strips the application package root path prefix from the path of the given file */ + private Path asApplicationRelativePath(File file) { + Path path = Path.fromString(file.toString()); + + Iterator<String> pathIterator = path.iterator(); + // Skip the path elements this shares with the root + for (Iterator<String> rootIterator = root.iterator(); rootIterator.hasNext(); ) { + String rootElement = rootIterator.next(); + String pathElement = pathIterator.next(); + if ( ! rootElement.equals(pathElement)) throw new RuntimeException("Assumption broken"); + } + // Build a path from the remaining + Path relative = Path.fromString(""); + while (pathIterator.hasNext()) + relative = relative.append(pathIterator.next()); + return relative; + } + + } + } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigApiTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigApiTest.java index 5419100fcf1..c0080091db6 100755 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigApiTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigApiTest.java @@ -12,10 +12,10 @@ import static org.hamcrest.CoreMatchers.is; /** * Tests ConfigSubscriber API, and the ConfigHandle class. * - * @author <a href="gv@yahoo-inc.com">Harald Musum</a> - * @since 5.1 + * @author Harald Musum */ public class ConfigApiTest { + private static final String CONFIG_ID = "raw:" + "times 1\n"; @Test diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index e32b410fbab..c9c6ef4b428 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -1,25 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.yahoo.config.application.api.ApplicationMetaData; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.io.reader.NamedReader; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinitionKey; -import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import org.apache.commons.io.IOUtils; -import java.io.*; -import java.util.*; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; /** * A class used for reading and writing application data to zookeeper. @@ -109,7 +116,6 @@ public class ZooKeeperClient { // gives lots and lots of debug output: // BasicConfigurator.configure(); try { logFine("zk operations: " + configCurator.getNumberOfOperations()); - logFine("zk operations: " + configCurator.getNumberOfOperations()); logFine("Feeding user def files into ZooKeeper"); writeUserDefs(app); logFine("zk operations: " + configCurator.getNumberOfOperations()); @@ -196,6 +202,9 @@ public class ZooKeeperClient { writeDir(app.getFile(Path.fromString(ApplicationPackage.ROUTINGTABLES_DIR)), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.ROUTINGTABLES_DIR), xmlFilter, true); + writeDir(app.getFile(ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR), + true); } private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java index 94707635950..8ac992821fd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java @@ -2,25 +2,24 @@ package com.yahoo.vespa.config.server.http; import com.google.inject.Inject; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.logging.AccessLog; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.tenant.Tenants; -import com.yahoo.config.provision.ApplicationId; import java.util.Optional; -import java.util.concurrent.Executor; /** * HTTP handler for a v2 getConfig operation * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ +// TODO: Make this API discoverable public class HttpGetConfigHandler extends HttpHandler { + private final RequestHandler requestHandler; public HttpGetConfigHandler(HttpHandler.Context ctx, RequestHandler requestHandler) { @@ -28,11 +27,12 @@ public class HttpGetConfigHandler extends HttpHandler { this.requestHandler = requestHandler; } + @SuppressWarnings("unused") // injected @Inject public HttpGetConfigHandler(HttpHandler.Context ctx, Tenants tenants) { this(ctx, tenants.defaultTenant().getRequestHandler()); } - + @Override public HttpResponse handleGET(HttpRequest req) { HttpConfigRequest request = HttpConfigRequest.createFromRequestV1(req); diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index adf01f161a4..b2349ed6dfc 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -12,31 +12,38 @@ import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.query.*; -import com.yahoo.search.query.profile.types.FieldType; -import com.yahoo.search.query.properties.PropertyMap; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.query.Model; +import com.yahoo.search.query.ParameterParser; +import com.yahoo.search.query.Presentation; +import com.yahoo.search.query.Properties; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.SessionId; +import com.yahoo.search.query.Sorting; +import com.yahoo.search.query.Sorting.AttributeSorter; +import com.yahoo.search.query.Sorting.FieldOrder; +import com.yahoo.search.query.Sorting.Order; +import com.yahoo.search.query.UniqueRequestId; +import com.yahoo.search.query.context.QueryContext; +import com.yahoo.search.query.profile.ModelObjectMap; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; import com.yahoo.search.query.properties.DefaultProperties; +import com.yahoo.search.query.properties.PropertyMap; import com.yahoo.search.query.properties.QueryProperties; import com.yahoo.search.query.properties.QueryPropertyAliases; import com.yahoo.search.query.properties.RequestContextProperties; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.federation.FederationSearcher; -import com.yahoo.search.query.Sorting.AttributeSorter; -import com.yahoo.search.query.Sorting.FieldOrder; -import com.yahoo.search.query.Sorting.Order; -import com.yahoo.search.query.context.QueryContext; -import com.yahoo.search.query.profile.ModelObjectMap; -import com.yahoo.search.query.profile.QueryProfileProperties; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.yql.NullItemException; import com.yahoo.search.yql.VespaSerializer; import com.yahoo.search.yql.YqlParser; - +import com.yahoo.yolean.Exceptions; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; @@ -45,7 +52,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; /** @@ -951,8 +957,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** * Return the HTTP request which caused this query. This will never be null * when running with queries from the network. - * (Except when running with deprecated code paths, in which case this will - * return null but getRequest() will not.) */ public HttpRequest getHttpRequest() { return httpRequest; } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 62eacaa0afe..2a36e30b95a 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -2,9 +2,13 @@ package com.yahoo.search.test; import com.yahoo.component.chain.Chain; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detection; import com.yahoo.language.detect.Detector; import com.yahoo.language.detect.Hint; +import com.yahoo.language.process.StemMode; +import com.yahoo.language.process.Token; import com.yahoo.language.simple.SimpleDetector; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.Index; @@ -65,6 +69,18 @@ public class QueryTestCase { assertNull(q.getModel().getDefaultIndex()); assertEquals("", q.properties().get("aParameter")); assertNull(q.properties().get("notSetParameter")); + + Query query = q; + String body = "a bb. ccc??!"; + Linguistics linguistics = new SimpleLinguistics(); + + AndItem and = new AndItem(); + for (Token token : linguistics.getTokenizer().tokenize(body, Language.ENGLISH, StemMode.SHORTEST, true)) { + if (token.isIndexable()) + and.addItem(new WordItem(token.getTokenString(), "body")); + } + query.getModel().getQueryTree().setRoot(and); + System.out.println(query); } // TODO: YQL work in progress (jon) diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java index ef94f67d6e6..2b2dcc90e41 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java @@ -1,16 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage.linguistics; -import java.util.HashMap; -import java.util.Map; - -import com.yahoo.document.annotation.*; +import com.yahoo.document.annotation.Annotation; +import com.yahoo.document.annotation.AnnotationTypes; +import com.yahoo.document.annotation.Span; +import com.yahoo.document.annotation.SpanList; +import com.yahoo.document.annotation.SpanTree; +import com.yahoo.document.annotation.SpanTrees; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.language.Linguistics; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.Token; import com.yahoo.language.process.Tokenizer; +import java.util.HashMap; +import java.util.Map; + import static com.yahoo.language.LinguisticsCase.toLowerCase; /** @@ -110,7 +115,7 @@ public class LinguisticsAnnotator { } return; } - if (!token.isIndexable()) { + if ( ! token.isIndexable()) { return; } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/AbstractApplication.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/AbstractApplication.java index e1bcef4c1e0..4075aa711c2 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/AbstractApplication.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/AbstractApplication.java @@ -39,7 +39,7 @@ import java.util.concurrent.TimeUnit; * } * </pre> * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public abstract class AbstractApplication implements Application { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java index 2c8d1c80b4f..907d2a050fb 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class ActiveContainer extends AbstractResource implements CurrentContainer { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java index 7556186c1a9..10b4c9f6508 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java @@ -29,7 +29,7 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen * @author bjorncs */ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, CurrentContainer { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapLoader.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapLoader.java index aa4fc4d1b18..a2e447dddf1 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapLoader.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapLoader.java @@ -2,15 +2,15 @@ package com.yahoo.jdisc.core; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public interface BootstrapLoader { - public void init(String bundleLocation, boolean privileged) throws Exception; + void init(String bundleLocation, boolean privileged) throws Exception; - public void start() throws Exception; + void start() throws Exception; - public void stop() throws Exception; + void stop() throws Exception; - public void destroy(); + void destroy(); } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/Main.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/Main.java index c3d5460ac38..ce091fa82d0 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/Main.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/Main.java @@ -10,7 +10,7 @@ import java.util.Arrays; import java.util.Collections; /** - * @author simon + * @author Simon Thoresen */ public class Main { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java index 17156d5659c..e72df8fc7e1 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java @@ -6,7 +6,6 @@ import org.osgi.framework.Bundle; import org.osgi.framework.ServiceReference; import org.osgi.service.log.LogService; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.logging.Handler; @@ -14,11 +13,11 @@ import java.util.logging.Level; import java.util.logging.LogRecord; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ class OsgiLogHandler extends Handler { - private static enum LogRecordProperty { + private enum LogRecordProperty { LEVEL, LOGGER_NAME, @@ -32,6 +31,7 @@ class OsgiLogHandler extends Handler { SOURCE_METHOD_NAME, THREAD_ID, THROWN + } private final static Map<String, LogRecordProperty> PROPERTY_MAP = createDictionary(LogRecordProperty.values()); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogManager.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogManager.java index 524863b1c91..5b0367c5a55 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogManager.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogManager.java @@ -13,7 +13,7 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ class OsgiLogManager implements LogService { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogService.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogService.java index 02f8c85b940..f51891cada4 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogService.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogService.java @@ -4,7 +4,7 @@ package com.yahoo.jdisc.core; import org.osgi.framework.*; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ class OsgiLogService { @@ -38,22 +38,22 @@ class OsgiLogService { return; } switch (event.getType()) { - case ServiceEvent.REGISTERED: - try { - activator.start(ctx); - } catch (Exception e) { - throw new RuntimeException("Exception thrown while starting " + - activator.getClass().getName() + ".", e); - } - break; - case ServiceEvent.UNREGISTERING: - try { - activator.stop(ctx); - } catch (Exception e) { - throw new RuntimeException("Exception thrown while stopping " + - activator.getClass().getName() + ".", e); - } - break; + case ServiceEvent.REGISTERED: + try { + activator.start(ctx); + } catch (Exception e) { + throw new RuntimeException("Exception thrown while starting " + + activator.getClass().getName() + ".", e); + } + break; + case ServiceEvent.UNREGISTERING: + try { + activator.stop(ctx); + } catch (Exception e) { + throw new RuntimeException("Exception thrown while stopping " + + activator.getClass().getName() + ".", e); + } + break; } } } diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java index 921c10a1821..1f839d0dc0a 100644 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java @@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult * @author bjorncs */ public class OsgiLogServiceIntegrationTest { diff --git a/linguistics/src/main/java/com/yahoo/language/process/Token.java b/linguistics/src/main/java/com/yahoo/language/process/Token.java index d730c5dd16b..73c0ac857ab 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Token.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Token.java @@ -2,9 +2,9 @@ package com.yahoo.language.process; /** - * Interface providing access to a single token produced by the tokenizer. + * A single token produced by the tokenizer. * - * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a> + * @author Mathias Mølster Lidal */ public interface Token { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java index 42945c59105..45f2b21343f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java @@ -14,6 +14,7 @@ import org.tensorflow.framework.SignatureDef; import org.tensorflow.framework.TensorInfo; import org.tensorflow.framework.TensorShapeProto; +import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; @@ -30,7 +31,7 @@ public class TensorFlowImporter { /** * Imports a saved TensorFlow model from a directory. - * The model should be saved as a pbtxt file. + * The model should be saved as a .pbtxt or .pb file. * The name of the model is taken as the db/pbtxt file name (not including the file ending). * * @param modelDir the directory containing the TensorFlow model files to import @@ -44,6 +45,10 @@ public class TensorFlowImporter { } } + public TensorFlowModel importModel(File modelDir) { + return importModel(modelDir.toString()); + } + /** Imports a TensorFlow model */ public TensorFlowModel importModel(SavedModelBundle model) { try { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java index fd981a14c3e..9fdc45ab3bc 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java @@ -69,6 +69,8 @@ public class TensorFlowModel { void output(String name, String expressionName) { outputs.put(name, expressionName); } void skippedOutput(String name, String reason) { skippedOutputs.put(name, reason); } + public String name() { return name; } + /** Returns the result this is part of */ TensorFlowModel owner() { return TensorFlowModel.this; } diff --git a/vespajlib/src/main/java/com/yahoo/path/Path.java b/vespajlib/src/main/java/com/yahoo/path/Path.java index 7389ca2af54..da55c6767d1 100644 --- a/vespajlib/src/main/java/com/yahoo/path/Path.java +++ b/vespajlib/src/main/java/com/yahoo/path/Path.java @@ -3,16 +3,15 @@ package com.yahoo.path; import com.google.common.annotations.Beta; +import java.io.File; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -// TODO: Remove and replace usage by java.nio.file.Path - /** * Represents a path represented by a list of elements. Immutable * - * @author lulf + * @author Ulf Lilleengen */ @Beta public final class Path { @@ -195,6 +194,8 @@ public final class Path { return new Path(delimiter); } + public File toFile() { return new File(toString()); } + @Override public int hashCode() { return elements.hashCode(); diff --git a/vespajlib/src/test/java/com/yahoo/text/Benchmark.java b/vespajlib/src/test/java/com/yahoo/text/Benchmark.java index b32927c2f5d..14eed6d80d4 100644 --- a/vespajlib/src/test/java/com/yahoo/text/Benchmark.java +++ b/vespajlib/src/test/java/com/yahoo/text/Benchmark.java @@ -14,7 +14,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ class Benchmark { @@ -103,4 +103,5 @@ class Benchmark { return new Benchmark(this); } } + } |