diff options
author | HÃ¥kon Hallingstad <hakon@oath.com> | 2018-01-22 23:57:23 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-01-22 23:57:23 +0100 |
commit | 1956fa077d9b218bee4df013d002cf1c20490f85 (patch) | |
tree | e74404b6e1a1587b158ef4e76c5889f2148daae4 | |
parent | 9ffa1b65e571c1878b3b5b26fc8f00d95efb9d3c (diff) | |
parent | bf7db444e31324b46a330a1046443a3ad227ab1f (diff) |
Merge pull request #4741 from vespa-engine/revert-4739-bratseth/correct-path-just-once
Revert "Correct path just once"
4 files changed, 38 insertions, 42 deletions
diff --git a/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java b/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java index 79ed538c57d..73110a06151 100644 --- a/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java +++ b/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java @@ -60,7 +60,7 @@ public class ApplicationPreprocessor { try { preprocessor.run(); System.out.println("Application preprocessed successfully. Preprocessed application stored in " + - new File(applicationDir, FilesApplicationPackage.preprocessed).getAbsolutePath()); + new File(applicationDir, ".preprocessed").getAbsolutePath()); } catch (Exception e) { System.err.println("Error validating application package: " + Exceptions.toMessageString(e)); System.exit(1); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 391d284a325..3388261c798 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -59,17 +59,6 @@ import static com.yahoo.text.Lowercase.toLowerCase; */ public class FilesApplicationPackage implements ApplicationPackage { - /** - * The name of the subdirectory (below the original application package root) - * where a preprocessed version of this application package is stored. - * As it happens, the config model is first created with an application package in this subdirectory, - * and later used backed by an application package which is not in this subdirectory. - * To enable model code to correct for this, this constant must be publicly known. - * - * All of this stuff is Very Unfortunate and should be fixed. -Jon - */ - public static final String preprocessed = ".preprocessed"; - private static final Logger log = Logger.getLogger(FilesApplicationPackage.class.getName()); private static final String META_FILE_NAME = ".applicationMetaData"; @@ -97,7 +86,7 @@ public class FilesApplicationPackage implements ApplicationPackage { * @return an Application package instance */ public static FilesApplicationPackage fromFile(File appDir, boolean includeSourceFiles) { - return new Builder(appDir).preprocessedDir(new File(appDir, preprocessed)) + return new Builder(appDir).preprocessedDir(new File(appDir, ".preprocessed")) .includeSourceFiles(includeSourceFiles) .build(); } @@ -676,7 +665,7 @@ public class FilesApplicationPackage implements ApplicationPackage { @Override public ApplicationPackage preprocess(Zone zone, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { IOUtils.recursiveDeleteDir(preprocessedDir); - IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> ! name.equals(preprocessed) && + IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> ! name.equals(".preprocessed") && ! name.equals(SERVICES) && ! name.equals(HOSTS) && ! name.equals(CONFIG_DEFINITIONS_DIR)); @@ -797,7 +786,7 @@ public class FilesApplicationPackage implements ApplicationPackage { } public FilesApplicationPackage build() { - return new FilesApplicationPackage(appDir, preprocessedDir.orElse(new File(appDir, preprocessed)), + return new FilesApplicationPackage(appDir, preprocessedDir.orElse(new File(appDir, ".preprocessed")), metaData.orElse(readMetaData(appDir)), includeSourceFiles); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java index c16d12f76d1..629aef7f56c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java @@ -4,7 +4,6 @@ package com.yahoo.searchdefinition.expressiontransforms; import com.google.common.base.Joiner; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankProfile; @@ -255,9 +254,13 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil // "tbf" ending for "typed binary format" - recognized by the nodes receiving the file: Path constantPath = constantsPath.append(name + ".tbf"); Path constantPathCorrected = constantPath; - if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) - && ! constantPath.elements().contains(FilesApplicationPackage.preprocessed)) { - constantPathCorrected = Path.fromString(FilesApplicationPackage.preprocessed).append(constantPath); + if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(".preprocessed")) { + log.info("Correcting TensorFlow constant path by prepending .preprocessed - alternative 1"); + constantPathCorrected = Path.fromString(".preprocessed").append(constantPath); + } + else if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(".preprocessed")) { + log.info("Correcting TensorFlow constant path by prepending .preprocessed - alternative 2"); + constantPathCorrected = Path.fromString(".preprocessed").append(constantPath); } // Remember the constant in a file we replicate in ZooKeeper diff --git a/vespajlib/src/main/java/com/yahoo/path/Path.java b/vespajlib/src/main/java/com/yahoo/path/Path.java index c466fe50d6f..da55c6767d1 100644 --- a/vespajlib/src/main/java/com/yahoo/path/Path.java +++ b/vespajlib/src/main/java/com/yahoo/path/Path.java @@ -2,14 +2,11 @@ package com.yahoo.path; import com.google.common.annotations.Beta; -import com.google.common.collect.ImmutableList; import java.io.File; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.stream.Collectors; /** * Represents a path represented by a list of elements. Immutable @@ -20,7 +17,7 @@ import java.util.stream.Collectors; public final class Path { private final String delimiter; - private final ImmutableList<String> elements; + private final List<String> elements = new ArrayList<>(); /** * Create an empty path. @@ -42,7 +39,7 @@ public final class Path { * @param elements a list of path elements */ private Path(List<String> elements, String delimiter) { - this.elements = ImmutableList.copyOf(elements); + this.elements.addAll(elements); this.delimiter = delimiter; } @@ -54,20 +51,26 @@ public final class Path { /** * Add path elements by splitting based on delimiter and appending to elements. */ - private static List<String> elementsOf(String path, String delimiter) { - return Arrays.stream(path.split(delimiter)).filter(e -> !"".equals(e)).collect(Collectors.toList()); + private void addElementsFromString(String path) { + String[] pathElements = path.split(delimiter); + if (pathElements != null) { + for (String elem : pathElements) { + if (!"".equals(elem)) { + elements.add(elem); + } + } + } } /** - * Append an element to the path. Returns a new path with the given path appended. - * - * @param path the path to append to this - * @return the new path + * Append an element to the path. Returns a new path with this element appended. + * @param name name of element to append. + * @return this, for chaining */ - public Path append(String path) { - List<String> newElements = new ArrayList<>(this.elements); - newElements.addAll(elementsOf(path, delimiter)); - return new Path(newElements, delimiter); + public Path append(String name) { + Path path = new Path(this); + path.addElementsFromString(name); + return path; } /** @@ -77,9 +80,9 @@ public final class Path { * @return a new path with argument appended to it. */ public Path append(Path path) { - List<String> newElements = new ArrayList<>(this.elements); - newElements.addAll(path.elements()); - return new Path(newElements, delimiter); + Path newPath = new Path(this); + newPath.elements.addAll(path.elements); + return newPath; } /** @@ -87,7 +90,9 @@ public final class Path { * @return the name */ public String getName() { - if (elements.isEmpty()) return ""; + if (elements.isEmpty()) { + return ""; + } return elements.get(elements.size() - 1); } @@ -136,9 +141,6 @@ public final class Path { public Iterator<String> iterator() { return elements.iterator(); } - /** Returns an immutable list of the elements of this path in order */ - public List<String> elements() { return elements; } - /** * Convert to string. * @@ -169,7 +171,9 @@ public final class Path { * @return a path object that may be used with the application package. */ public static Path fromString(String path, String delimiter) { - return new Path(elementsOf(path, delimiter), delimiter); + Path pathObj = new Path(delimiter); + pathObj.addElementsFromString(path); + return pathObj; } /** |