From 7e2e38d17a51d0ca93dc74b8e7e0d34c5eeb19af Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 11 Jan 2018 13:32:09 +0100 Subject: Use constant tensor files WIP --- .../config/application/api/ApplicationPackage.java | 6 ++-- .../config/application/api/ComponentInfo.java | 3 +- .../com/yahoo/searchdefinition/RankProfile.java | 4 +-- .../yahoo/searchdefinition/RankingConstant.java | 26 +++++++++++++----- .../java/com/yahoo/searchdefinition/Search.java | 10 +++---- .../TensorFlowFeatureConverter.java | 26 ++++++++++++++++-- .../validation/RankingConstantsValidator.java | 14 +++++----- .../vespa/model/search/AbstractSearchCluster.java | 27 +++++++++--------- .../yahoo/vespa/model/search/DocumentDatabase.java | 32 ++++++++++------------ .../com/yahoo/vespa/model/utils/FileSender.java | 1 - .../RankingExpressionWithTensorFlowTestCase.java | 3 +- .../config/server/deploy/ZooKeeperClient.java | 2 +- .../filedistribution/ApplicationFileManager.java | 2 ++ .../filedistribution/FileDistributionProvider.java | 3 +- .../server/zookeeper/ZKApplicationPackage.java | 14 ++++------ .../src/main/java/com/yahoo/tensor/Tensor.java | 6 ++++ 16 files changed, 109 insertions(+), 70 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 7506c884715..aca7b595249 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -4,7 +4,6 @@ package com.yahoo.config.application.api; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; -import com.yahoo.path.Path; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; @@ -42,7 +41,7 @@ import java.util.jar.JarFile; */ public interface ApplicationPackage { - // Caution!! If you add something here it must probably also be added to ZooKeeperClient.feedZKAppPkg + // Caution!! If you add something here it must probably also be added to ZooKeeperClient.write(applicationPackage) String HOSTS = "hosts.xml"; String SERVICES = "services.xml"; @@ -54,6 +53,7 @@ public interface ApplicationPackage { String DOCPROCCHAINS_DIR = "docproc/chains"; String PROCESSORCHAINS_DIR = "processor/chains"; String ROUTINGTABLES_DIR = "routing/tables"; + String MODELS_DIR = "models"; // NOTE: this directory is created in serverdb during deploy, and should not exist in the original user application /** Do not use */ @@ -271,7 +271,7 @@ public interface ApplicationPackage { * * @return A new application package instance pointing to a new location */ - default ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ruleConfigDeriver, DeployLogger logger) + default ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ruleConfigDeriver, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { throw new UnsupportedOperationException("This application package does not support preprocessing"); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ComponentInfo.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ComponentInfo.java index fc6f09cef24..3dd10054d13 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ComponentInfo.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ComponentInfo.java @@ -4,9 +4,10 @@ package com.yahoo.config.application.api; /** * Describes a component residing in the components directory. - *

TODO: add support for component versions.

+ * * @author tonytv */ +// TODO: add support for component versions. public class ComponentInfo { final String pathRelativeToAppDir; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index ec3100bc6b9..bacff94d776 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -39,7 +39,7 @@ public class RankProfile implements Serializable, Cloneable { private final String name; /** The search definition owning this profile, or null if none */ - private Search search=null; + private Search search = null; /** The name of the rank profile inherited by this */ private String inheritedName = null; @@ -51,7 +51,7 @@ public class RankProfile implements Serializable, Cloneable { protected Set rankSettings = new java.util.LinkedHashSet<>(); /** The ranking expression to be used for first phase */ - private RankingExpression firstPhaseRanking= null; + private RankingExpression firstPhaseRanking = null; /** The ranking expression to be used for second phase */ private RankingExpression secondPhaseRanking = null; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java index 16e57ee913d..c65e0fad1c7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java @@ -6,7 +6,7 @@ import com.yahoo.tensor.TensorType; import java.util.Objects; /** - * Represents a global ranking constant (declared in a .sd file) + * Represents a global ranking constant * * @author arnej */ @@ -16,23 +16,35 @@ public class RankingConstant { private final String name; private TensorType tensorType = null; private String fileName = null; - private String fileRef = ""; + private String fileReference = ""; public RankingConstant(String name) { this.name = name; } - public void setFileName(String fileName) { + public RankingConstant(String name, TensorType type, String fileName) { + this(name); + this.tensorType = type; + this.fileName = fileName; + validate(); + } + + public void setFileName(String fileName) { Objects.requireNonNull(fileName, "Filename cannot be null"); - this.fileName = fileName; + this.fileName = fileName; } - public void setFileReference(String fileRef) { this.fileRef = fileRef; } + /** + * Set the internally generated reference to this file used to identify this instance of the file for + * file distribution. + */ + public void setFileReference(String fileReference) { this.fileReference = fileReference; } + public void setType(TensorType tensorType) { this.tensorType = tensorType; } public String getName() { return name; } public String getFileName() { return fileName; } - public String getFileReference() { return fileRef; } + public String getFileReference() { return fileReference; } public TensorType getTensorType() { return tensorType; } public String getType() { return tensorType.toString(); } @@ -47,7 +59,7 @@ public class RankingConstant { StringBuilder b = new StringBuilder(); b.append("constant '").append(name) .append("' from file '").append(fileName) - .append("' with ref '").append(fileRef) + .append("' with ref '").append(fileReference) .append("' of type '").append(tensorType) .append("'"); return b.toString(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index 7baa3eb170f..bd7b8ce6e15 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -58,7 +58,7 @@ public class Search implements Serializable, ImmutableSearch { // Field sets private FieldSets fieldSets = new FieldSets(); - + // Whether or not this object has been processed. private boolean processed; @@ -162,13 +162,13 @@ public class Search implements Serializable, ImmutableSearch { docType = document; } - public void addRankingConstant(RankingConstant rConstant) { - rConstant.validate(); - String name = rConstant.getName(); + public void addRankingConstant(RankingConstant constant) { + constant.validate(); + String name = constant.getName(); if (rankingConstants.get(name) != null) { throw new IllegalArgumentException("Ranking constant '"+name+"' defined twice"); } - rankingConstants.put(name, rConstant); + rankingConstants.put(name, constant); } public Iterable getRankingConstants() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java index d05027dda39..a36384ce6f2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java @@ -1,17 +1,22 @@ package com.yahoo.searchdefinition.expressiontransforms; import com.google.common.base.Joiner; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowModel; import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowImporter; +import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowModel; import com.yahoo.searchlib.rankingexpression.rule.Arguments; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.tensor.Tensor; +import java.io.File; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -48,7 +53,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer tensorFlowImporter.importModel(modelPath)); // Find the specified expression @@ -58,7 +63,10 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer context.rankProfile().addConstantTensor(k, new TensorValue(v))); + if (1==1) + result.constants().forEach((k, v) -> context.rankProfile().addConstantTensor(k, new TensorValue(v))); + else // correct way, disabled for now + result.constants().forEach((k, v) -> transformConstant(modelPath, context.rankProfile(), k, v)); return expression.getRoot(); } @@ -120,6 +128,18 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer backends) { for (SearchDefinitionSpec sds : localSDS) { - for (RankingConstant rc : sds.getSearchDefinition().getSearch().getRankingConstants()) { - FileReference reference = FileSender.sendFileToServices(rc.getFileName(), backends); - rc.setFileReference(reference.value()); + for (RankingConstant constant : sds.getSearchDefinition().getSearch().getRankingConstants()) { + FileReference reference = FileSender.sendFileToServices(constant.getFileName(), backends); + constant.setFileReference(reference.value()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java index 4d4aea93a36..32548039fdd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java @@ -3,19 +3,17 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.RankingConstant; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.ImportedFieldsConfig; -import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.config.search.IndexschemaConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.config.search.SummarymapConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.summary.JuniperrcConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; -import com.yahoo.config.FileReference; -import com.yahoo.vespa.model.utils.FileSender; /** * Represents a document database and the backend configuration needed for this database. @@ -59,17 +57,17 @@ public class DocumentDatabase extends AbstractConfigProducer implements public void getConfig(IndexInfoConfig.Builder builder) { derivedCfg.getIndexInfo().getConfig(builder); } - + @Override public void getConfig(IlscriptsConfig.Builder builder) { derivedCfg.getIndexingScript().getConfig(builder); } - + @Override public void getConfig(AttributesConfig.Builder builder) { derivedCfg.getAttributeFields().getConfig(builder); } - + @Override public void getConfig(RankProfilesConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); @@ -77,15 +75,15 @@ public class DocumentDatabase extends AbstractConfigProducer implements @Override public void getConfig(RankingConstantsConfig.Builder builder) { - for (RankingConstant rConstant : derivedCfg.getSearch().getRankingConstants()) { - if ("".equals(rConstant.getFileReference())) { - System.err.println("INVALID rank constant "+rConstant.getName()+" [missing file reference]"); + for (RankingConstant constant : derivedCfg.getSearch().getRankingConstants()) { + if ("".equals(constant.getFileReference())) { + System.err.println("INVALID rank constant "+constant.getName()+" [missing file reference]"); // TODO: Throw or log warning continue; } builder.constant(new RankingConstantsConfig.Constant.Builder() - .name(rConstant.getName()) - .fileref(rConstant.getFileReference()) - .type(rConstant.getType())); + .name(constant.getName()) + .fileref(constant.getFileReference()) + .type(constant.getType())); } } @@ -93,17 +91,17 @@ public class DocumentDatabase extends AbstractConfigProducer implements public void getConfig(IndexschemaConfig.Builder builder) { derivedCfg.getIndexSchema().getConfig(builder); } - + @Override public void getConfig(JuniperrcConfig.Builder builder) { derivedCfg.getJuniperrc().getConfig(builder); } - + @Override public void getConfig(SummarymapConfig.Builder builder) { derivedCfg.getSummaryMap().getConfig(builder); } - + @Override public void getConfig(SummaryConfig.Builder builder) { derivedCfg.getSummaries().getConfig(builder); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 605c87912ac..413363d7b0d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -17,7 +17,6 @@ import java.util.*; * Utility methods for sending files to a collection of nodes. * * @author gjoranv - * @since 5.1.9 */ public class FileSender implements Serializable { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 8fcd821adfd..3ec621618e5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -15,7 +15,8 @@ import static org.junit.Assert.fail; */ public class RankingExpressionWithTensorFlowTestCase { - private final String modelDirectory = "src/test/integration/tensorflow/mnist_softmax/saved"; + // The "../" is to escape the "models/" element prepended to the path + private final String modelDirectory = "../src/test/integration/tensorflow/mnist_softmax/saved"; private final String vespaExpression = "join(rename(reduce(join(Placeholder, rename(constant(Variable), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(Variable_1), d0, d1), f(a,b)(a + b))"; @Test diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 69266620e45..e32b410fbab 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -100,7 +100,7 @@ public class ZooKeeperClient { } /** - * Feeds def files and user config into ZK. + * Writes def files and user config into ZK. * * @param app the application package to feed to zookeeper */ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java index 0d1aae97690..82535143c89 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java @@ -5,6 +5,7 @@ import com.yahoo.config.FileReference; import java.io.File; public class ApplicationFileManager implements AddFileInterface { + private final File applicationDir; private final FileDirectory master; @@ -22,4 +23,5 @@ public class ApplicationFileManager implements AddFileInterface { public FileReference addFile(String relativePath) { return master.addFile(new File(applicationDir, relativePath)); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java index cd3f0f7f167..6c2da338ef0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java @@ -21,6 +21,7 @@ public class FileDistributionProvider { private final FileDistribution fileDistribution; static private class ManagerWrapper implements AddFileInterface { + private final FileDistributionManager manager; ManagerWrapper(FileDistributionManager manager) { this.manager = manager; @@ -45,7 +46,7 @@ public class FileDistributionProvider { zooKeepersSpec, applicationId, fileDistributionLock); this.fileDistribution = new CombinedLegacyDistribution(supervisor, new FileDBHandler(manager), disableFileDistributor); this.fileRegistry = new CombinedLegacyRegistry(new FileDBRegistry(new ManagerWrapper(manager)), - new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory()))); + new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory()))); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index e4933dc84c4..6643e9b3370 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -61,9 +61,9 @@ public class ZKApplicationPackage implements ApplicationPackage { return Optional.of(readAllocatedHosts(allocatedHostsPath, nodeFlavors)); } - /** + /** * Reads allocated hosts at the given node. - * + * * @return the allocated hosts at this node or empty if there is no data at this path */ private AllocatedHosts readAllocatedHosts(String allocatedHostsPath, Optional nodeFlavors) { @@ -79,11 +79,9 @@ public class ZKApplicationPackage implements ApplicationPackage { if (fileRegistryNodes.isEmpty()) { fileRegistryMap.put(legacyVersion, importFileRegistry(fileRegistryNode)); } else { - fileRegistryNodes.stream() - .forEach(version -> { + fileRegistryNodes.forEach(version -> fileRegistryMap.put(com.yahoo.config.provision.Version.fromString(version), - importFileRegistry(Joiner.on("/").join(fileRegistryNode, version))); - }); + importFileRegistry(Joiner.on("/").join(fileRegistryNode, version)))); } } @@ -155,7 +153,7 @@ public class ZKApplicationPackage implements ApplicationPackage { private Optional getPreGeneratedFileRegistry(com.yahoo.config.provision.Version vespaVersion) { // Assumes at least one file registry, which we always have. Optional fileRegistry = Optional.ofNullable(fileRegistryMap.get(vespaVersion)); - if (!fileRegistry.isPresent()) { + if ( ! fileRegistry.isPresent()) { fileRegistry = Optional.of(fileRegistryMap.values().iterator().next()); } return fileRegistry; @@ -243,7 +241,7 @@ public class ZKApplicationPackage implements ApplicationPackage { List components = new ArrayList<>(); PreGeneratedFileRegistry fileRegistry = getPreGeneratedFileRegistry(vespaVersion).get(); for (String path : fileRegistry.getPaths()) { - if (path.startsWith(FilesApplicationPackage.COMPONENT_DIR + File.separator) && path.endsWith(".jar")) { + if (path.startsWith(ApplicationPackage.COMPONENT_DIR + File.separator) && path.endsWith(".jar")) { ComponentInfo component = new ComponentInfo(path); components.add(component); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 0c948f1fbee..2255cf82338 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -182,16 +182,22 @@ public interface Tensor { default Tensor notEqual(Tensor argument) { return join(argument, (a, b) -> ( a != b ? 1.0 : 0.0)); } default Tensor approxEqual(Tensor argument) { return join(argument, (a, b) -> ( approxEquals(a,b) ? 1.0 : 0.0)); } + default Tensor avg() { return avg(Collections.emptyList()); } default Tensor avg(String dimension) { return avg(Collections.singletonList(dimension)); } default Tensor avg(List dimensions) { return reduce(Reduce.Aggregator.avg, dimensions); } + default Tensor count() { return count(Collections.emptyList()); } default Tensor count(String dimension) { return count(Collections.singletonList(dimension)); } default Tensor count(List dimensions) { return reduce(Reduce.Aggregator.count, dimensions); } + default Tensor max() { return max(Collections.emptyList()); } default Tensor max(String dimension) { return max(Collections.singletonList(dimension)); } default Tensor max(List dimensions) { return reduce(Reduce.Aggregator.max, dimensions); } + default Tensor min() { return min(Collections.emptyList()); } default Tensor min(String dimension) { return min(Collections.singletonList(dimension)); } default Tensor min(List dimensions) { return reduce(Reduce.Aggregator.min, dimensions); } + default Tensor prod() { return prod(Collections.emptyList()); } default Tensor prod(String dimension) { return prod(Collections.singletonList(dimension)); } default Tensor prod(List dimensions) { return reduce(Reduce.Aggregator.prod, dimensions); } + default Tensor sum() { return sum(Collections.emptyList()); } default Tensor sum(String dimension) { return sum(Collections.singletonList(dimension)); } default Tensor sum(List dimensions) { return reduce(Reduce.Aggregator.sum, dimensions); } -- cgit v1.2.3