diff options
4 files changed, 42 insertions, 38 deletions
diff --git a/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java b/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java index 73110a06151..79ed538c57d 100644 --- a/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java +++ b/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java @@ -60,7 +60,7 @@ public class ApplicationPreprocessor { try { preprocessor.run(); System.out.println("Application preprocessed successfully. Preprocessed application stored in " + - new File(applicationDir, ".preprocessed").getAbsolutePath()); + new File(applicationDir, FilesApplicationPackage.preprocessed).getAbsolutePath()); } catch (Exception e) { System.err.println("Error validating application package: " + Exceptions.toMessageString(e)); System.exit(1); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 3388261c798..391d284a325 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -59,6 +59,17 @@ import static com.yahoo.text.Lowercase.toLowerCase; */ public class FilesApplicationPackage implements ApplicationPackage { + /** + * The name of the subdirectory (below the original application package root) + * where a preprocessed version of this application package is stored. + * As it happens, the config model is first created with an application package in this subdirectory, + * and later used backed by an application package which is not in this subdirectory. + * To enable model code to correct for this, this constant must be publicly known. + * + * All of this stuff is Very Unfortunate and should be fixed. -Jon + */ + public static final String preprocessed = ".preprocessed"; + private static final Logger log = Logger.getLogger(FilesApplicationPackage.class.getName()); private static final String META_FILE_NAME = ".applicationMetaData"; @@ -86,7 +97,7 @@ public class FilesApplicationPackage implements ApplicationPackage { * @return an Application package instance */ public static FilesApplicationPackage fromFile(File appDir, boolean includeSourceFiles) { - return new Builder(appDir).preprocessedDir(new File(appDir, ".preprocessed")) + return new Builder(appDir).preprocessedDir(new File(appDir, preprocessed)) .includeSourceFiles(includeSourceFiles) .build(); } @@ -665,7 +676,7 @@ public class FilesApplicationPackage implements ApplicationPackage { @Override public ApplicationPackage preprocess(Zone zone, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { IOUtils.recursiveDeleteDir(preprocessedDir); - IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> ! name.equals(".preprocessed") && + IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> ! name.equals(preprocessed) && ! name.equals(SERVICES) && ! name.equals(HOSTS) && ! name.equals(CONFIG_DEFINITIONS_DIR)); @@ -786,7 +797,7 @@ public class FilesApplicationPackage implements ApplicationPackage { } public FilesApplicationPackage build() { - return new FilesApplicationPackage(appDir, preprocessedDir.orElse(new File(appDir, ".preprocessed")), + return new FilesApplicationPackage(appDir, preprocessedDir.orElse(new File(appDir, preprocessed)), metaData.orElse(readMetaData(appDir)), includeSourceFiles); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java index 629aef7f56c..c16d12f76d1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.expressiontransforms; import com.google.common.base.Joiner; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankProfile; @@ -254,13 +255,9 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil // "tbf" ending for "typed binary format" - recognized by the nodes receiving the file: Path constantPath = constantsPath.append(name + ".tbf"); Path constantPathCorrected = constantPath; - if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(".preprocessed")) { - log.info("Correcting TensorFlow constant path by prepending .preprocessed - alternative 1"); - constantPathCorrected = Path.fromString(".preprocessed").append(constantPath); - } - else if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(".preprocessed")) { - log.info("Correcting TensorFlow constant path by prepending .preprocessed - alternative 2"); - constantPathCorrected = Path.fromString(".preprocessed").append(constantPath); + if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) + && ! constantPath.elements().contains(FilesApplicationPackage.preprocessed)) { + constantPathCorrected = Path.fromString(FilesApplicationPackage.preprocessed).append(constantPath); } // Remember the constant in a file we replicate in ZooKeeper diff --git a/vespajlib/src/main/java/com/yahoo/path/Path.java b/vespajlib/src/main/java/com/yahoo/path/Path.java index da55c6767d1..c466fe50d6f 100644 --- a/vespajlib/src/main/java/com/yahoo/path/Path.java +++ b/vespajlib/src/main/java/com/yahoo/path/Path.java @@ -2,11 +2,14 @@ package com.yahoo.path; import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableList; import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.stream.Collectors; /** * Represents a path represented by a list of elements. Immutable @@ -17,7 +20,7 @@ import java.util.List; public final class Path { private final String delimiter; - private final List<String> elements = new ArrayList<>(); + private final ImmutableList<String> elements; /** * Create an empty path. @@ -39,7 +42,7 @@ public final class Path { * @param elements a list of path elements */ private Path(List<String> elements, String delimiter) { - this.elements.addAll(elements); + this.elements = ImmutableList.copyOf(elements); this.delimiter = delimiter; } @@ -51,26 +54,20 @@ public final class Path { /** * Add path elements by splitting based on delimiter and appending to elements. */ - private void addElementsFromString(String path) { - String[] pathElements = path.split(delimiter); - if (pathElements != null) { - for (String elem : pathElements) { - if (!"".equals(elem)) { - elements.add(elem); - } - } - } + private static List<String> elementsOf(String path, String delimiter) { + return Arrays.stream(path.split(delimiter)).filter(e -> !"".equals(e)).collect(Collectors.toList()); } /** - * Append an element to the path. Returns a new path with this element appended. - * @param name name of element to append. - * @return this, for chaining + * Append an element to the path. Returns a new path with the given path appended. + * + * @param path the path to append to this + * @return the new path */ - public Path append(String name) { - Path path = new Path(this); - path.addElementsFromString(name); - return path; + public Path append(String path) { + List<String> newElements = new ArrayList<>(this.elements); + newElements.addAll(elementsOf(path, delimiter)); + return new Path(newElements, delimiter); } /** @@ -80,9 +77,9 @@ public final class Path { * @return a new path with argument appended to it. */ public Path append(Path path) { - Path newPath = new Path(this); - newPath.elements.addAll(path.elements); - return newPath; + List<String> newElements = new ArrayList<>(this.elements); + newElements.addAll(path.elements()); + return new Path(newElements, delimiter); } /** @@ -90,9 +87,7 @@ public final class Path { * @return the name */ public String getName() { - if (elements.isEmpty()) { - return ""; - } + if (elements.isEmpty()) return ""; return elements.get(elements.size() - 1); } @@ -141,6 +136,9 @@ public final class Path { public Iterator<String> iterator() { return elements.iterator(); } + /** Returns an immutable list of the elements of this path in order */ + public List<String> elements() { return elements; } + /** * Convert to string. * @@ -171,9 +169,7 @@ public final class Path { * @return a path object that may be used with the application package. */ public static Path fromString(String path, String delimiter) { - Path pathObj = new Path(delimiter); - pathObj.addElementsFromString(path); - return pathObj; + return new Path(elementsOf(path, delimiter), delimiter); } /** |