diff options
author | Jon Bratseth <bratseth@verizonmedia.com> | 2020-03-17 11:41:59 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@verizonmedia.com> | 2020-03-17 11:41:59 +0100 |
commit | 79f4cb5be4a6719d0b11cd39c3f44af2849903b8 (patch) | |
tree | 58d830381f0e6f05309d49d4b8f5c88aa2545db9 /config-application-package | |
parent | ca1eae82829cd9f2d270b2b788432a4461c1dc3e (diff) |
Revert "Merge pull request #12565 from vespa-engine/revert-12563-bratseth/search-to-schema"
This reverts commit ea5fe1bf8c0d389a3964d2e5c490994920d21a4e, reversing
changes made to 7e92c3852e893d09f0aa821e04e4e9dbe83b4ab2.
Diffstat (limited to 'config-application-package')
3 files changed, 76 insertions, 95 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java index 32054b1ad9a..2b7aa5ab6e1 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java @@ -59,8 +59,7 @@ public class AppSubDirs { return configDefs.first; } - public File searchdefinitions() { - return searchdefinitions.first; - } + @Deprecated // Remove after March 2020 + public File searchdefinitions() { return searchdefinitions.first; } } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java index 74ade9d8e14..b0c4f74f9f6 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java @@ -35,20 +35,11 @@ public class ApplicationPackageXmlFilesValidator { return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), vespaVersion); } - @SuppressWarnings("deprecation") public void checkApplication() throws IOException { validate(validators.servicesXmlValidator(), servicesFileName()); validateOptional(validators.hostsXmlValidator(), FilesApplicationPackage.HOSTS); validateOptional(validators.deploymentXmlValidator(), FilesApplicationPackage.DEPLOYMENT_FILE.getName()); validateOptional(validators.validationOverridesXmlValidator(), FilesApplicationPackage.VALIDATION_OVERRIDES.getName()); - - if (appDirs.searchdefinitions().exists()) { - if (FilesApplicationPackage.getSearchDefinitionFiles(appDirs.root()).isEmpty()) { - throw new IllegalArgumentException("Application package in " + appDirs.root() + - " must contain at least one search definition (.sd) file when directory searchdefinitions/ exists."); - } - } - validateRouting(appDirs.routingtables); } @@ -71,7 +62,6 @@ public class ApplicationPackageXmlFilesValidator { validator.validate(appDirs.file(filename)); } - @SuppressWarnings("deprecation") private String servicesFileName() { String servicesFile = FilesApplicationPackage.SERVICES; if ( ! appDirs.file(servicesFile).exists()) { diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 8c3ccdc4c0b..5860c365947 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -49,6 +49,7 @@ import java.security.MessageDigest; import java.util.*; import java.util.jar.JarFile; import java.util.logging.Logger; +import java.util.stream.Stream; import static com.yahoo.text.Lowercase.toLowerCase; @@ -336,77 +337,73 @@ public class FilesApplicationPackage implements ApplicationPackage { public static Map<String, String> allSdsFromDocprocBundlesAndClasspath(File appDir) throws IOException { File dpChains = new File(appDir, ApplicationPackage.COMPONENT_DIR); if (!dpChains.exists() || !dpChains.isDirectory()) return Collections.emptyMap(); - List<String> usedNames = new ArrayList<>(); - Map<String, String> ret = new LinkedHashMap<>(); + Set<String> usedNames = new HashSet<>(); + Map<String, String> schemas = new LinkedHashMap<>(); // try classpath first - allSdsOnClassPath(usedNames, ret); + allSdsOnClassPath(usedNames, schemas); - for (File bundle : dpChains.listFiles(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".jar"); - }})) { + for (File bundle : dpChains.listFiles((File dir, String name) -> name.endsWith(".jar"))) { for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", new JarFile(bundle)).entrySet()) { String sdName = entry.getKey(); if (usedNames.contains(sdName)) { - throw new IllegalArgumentException("The search definition name '"+sdName+"' used in bundle '"+ - bundle.getName()+"' is already used in classpath or previous bundle."); + throw new IllegalArgumentException("The search definition name '" + sdName + "' used in bundle '"+ + bundle.getName()+ "' is already used in classpath or previous bundle."); } usedNames.add(sdName); String sdPayload = entry.getValue(); - ret.put(sdName, sdPayload); + schemas.put(sdName, sdPayload); } } - return ret; + return schemas; } - private static void allSdsOnClassPath(List<String> usedNames, Map<String, String> ret) throws IOException { - Enumeration<java.net.URL> resources = FilesApplicationPackage.class.getClassLoader().getResources(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); - - while(resources.hasMoreElements()) { - URL resource = resources.nextElement(); - - String protocol = resource.getProtocol(); + private static void allSdsOnClassPath(Set<String> usedNames, Map<String, String> schemas) { + ClassLoader cl = FilesApplicationPackage.class.getClassLoader(); + Stream<URL> resources = Stream.concat(cl.resources(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()), + cl.resources(ApplicationPackage.SCHEMAS_DIR.getRelative())); + resources.forEach(resource -> addSchemaFrom(resource, schemas, usedNames)); + } - if ("file".equals(protocol)) { - File file; - try { - file = new File(resource.toURI()); - } catch (URISyntaxException e) { - continue; - } - // only interested in directories - if (file.isDirectory()) { - List<File> sdFiles = getSearchDefinitionFiles(file); - for (File sdFile : sdFiles) { - String sdName = sdFile.getName(); + private static void addSchemaFrom(URL resource, Map<String, String> schemas, Set<String> usedNames) { + try { + switch (resource.getProtocol()) { + case "file": + File file = new File(resource.toURI()); + if (file.isDirectory()) { + List<File> sdFiles = getSearchDefinitionFiles(file); + for (File sdFile : sdFiles) { + String sdName = sdFile.getName(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definition name '" + sdName + + "' found in classpath already used earlier in classpath."); + } + usedNames.add(sdName); + String contents = IOUtils.readAll(new FileReader(sdFile)); + schemas.put(sdFile.getName(), contents); + } + } + break; + case "jar": + JarURLConnection jarConnection = (JarURLConnection) resource.openConnection(); + JarFile jarFile = jarConnection.getJarFile(); + for (Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", jarFile).entrySet()) { + String sdName = entry.getKey(); if (usedNames.contains(sdName)) { - throw new IllegalArgumentException("The search definition name '"+sdName+ - "' found in classpath already used earlier in classpath."); + throw new IllegalArgumentException("The search definitions name '" + sdName + + "' used in bundle '" + jarFile.getName() + "' " + + "is already used in classpath or previous bundle."); } usedNames.add(sdName); - String contents = IOUtils.readAll(new FileReader(sdFile)); - ret.put(sdFile.getName(), contents); - } - } - } - else if ("jar".equals(protocol)) { - JarURLConnection jarConnection = (JarURLConnection) resource.openConnection(); - JarFile jarFile = jarConnection.getJarFile(); - for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", jarFile).entrySet()) { - String sdName = entry.getKey(); - if (usedNames.contains(sdName)) { - throw new IllegalArgumentException("The search definitions name '"+sdName+ - "' used in bundle '"+jarFile.getName()+"' " + - "is already used in classpath or previous bundle."); + String sdPayload = entry.getValue(); + schemas.put(sdName, sdPayload); } - usedNames.add(sdName); - String sdPayload = entry.getValue(); - ret.put(sdName, sdPayload); - } + break; } } + catch (IOException | URISyntaxException e) { + throw new IllegalArgumentException("Could not read schema from '" + resource + "'", e); + } } /** @@ -462,11 +459,7 @@ public class FilesApplicationPackage implements ApplicationPackage { if (! configDefsDir.isDirectory()) return; log.log(LogLevel.DEBUG, "Getting all config definitions from '" + configDefsDir + "'"); - for (File def : configDefsDir.listFiles( - new FilenameFilter() { @Override public boolean accept(File dir, String name) { // TODO: Fix - return name.matches(".*\\.def");}})) { - - log.log(LogLevel.DEBUG, "Processing config definition '" + def + "'"); + for (File def : configDefsDir.listFiles((File dir, String name) -> name.matches(".*\\.def"))) { String[] nv = def.getName().split("\\.def"); ConfigDefinitionKey key; try { @@ -521,30 +514,25 @@ public class FilesApplicationPackage implements ApplicationPackage { } } - //Only intended for DeployProcessor, others should use the member version static List<File> getSearchDefinitionFiles(File appDir) { - //The dot is escaped later in this method: - assert (ApplicationPackage.SD_NAME_SUFFIX.charAt(0) == '.'); + List<File> schemaFiles = new ArrayList<>(); + + File sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + if (sdDir.isDirectory()) + schemaFiles.addAll(Arrays.asList(sdDir.listFiles((dir, name) -> name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX)))); - List<File> ret = new ArrayList<>(); - File sdDir; + sdDir = new File(appDir, ApplicationPackage.SCHEMAS_DIR.getRelative()); + if (sdDir.isDirectory()) + schemaFiles.addAll(Arrays.asList(sdDir.listFiles((dir, name) -> name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX)))); - sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); - if (!sdDir.isDirectory()) { - return ret; - } - ret.addAll(Arrays.asList( - sdDir.listFiles( - new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX);}}))); - return ret; + return schemaFiles; } public List<File> getSearchDefinitionFiles() { return getSearchDefinitionFiles(appDir); } - //Only for use by deploy processor + // Only for use by deploy processor public static List<Component> getComponents(File appDir) { List<Component> components = new ArrayList<>(); for (Bundle bundle : Bundle.getBundles(new File(appDir, ApplicationPackage.COMPONENT_DIR))) { @@ -622,6 +610,7 @@ public class FilesApplicationPackage implements ApplicationPackage { public Bundle getBundle() { return bundle; } + } // class Component /** @@ -643,12 +632,14 @@ public class FilesApplicationPackage implements ApplicationPackage { } private File expressionFileNameToFile(String name) { - File expressionFile = new File(name); - if (expressionFile.isAbsolute()) { + if (new File(name).isAbsolute()) throw new IllegalArgumentException("Absolute path to ranking expression file is not allowed: " + name); - } - File sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); - return new File(sdDir, name); + + File sdDir = new File(appDir, ApplicationPackage.SCHEMAS_DIR.getRelative()); + File expressionFile = new File(sdDir, name); + if ( !expressionFile.exists()) + expressionFile = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + return expressionFile; } @Override @@ -700,9 +691,8 @@ public class FilesApplicationPackage implements ApplicationPackage { ! name.equals(HOSTS) && ! name.equals(CONFIG_DEFINITIONS_DIR)); preprocessXML(new File(preprocessedDir, SERVICES), getServicesFile(), zone); - if (getHostsFile().exists()) { + if (getHostsFile().exists()) preprocessXML(new File(preprocessedDir, HOSTS), getHostsFile(), zone); - } FilesApplicationPackage preprocessed = FilesApplicationPackage.fromFile(preprocessedDir, includeSourceFiles); preprocessed.copyUserDefsIntoApplication(); return preprocessed; @@ -742,11 +732,12 @@ public class FilesApplicationPackage implements ApplicationPackage { /** * Adds the given path to the digest, or does nothing if path is neither file nor dir + * * @param path path to add to message digest * @param suffix only files with this suffix are considered * @param digest the {link @MessageDigest} to add the file paths to * @param recursive whether to recursively find children in the paths - * @param fullPathNames Whether to include the full paths in checksum or only the names + * @param fullPathNames whether to include the full paths in checksum or only the names * @throws java.io.IOException if adding path to digest fails when reading files from path */ private static void addPathToDigest(File path, String suffix, MessageDigest digest, boolean recursive, boolean fullPathNames) throws IOException { @@ -773,16 +764,17 @@ public class FilesApplicationPackage implements ApplicationPackage { } private static final int MD5_BUFFER_SIZE = 65536; + private static void addToDigest(InputStream is, MessageDigest digest) throws IOException { - if (is==null) return; + if (is == null) return; byte[] buffer = new byte[MD5_BUFFER_SIZE]; int i; do { - i=is.read(buffer); + i = is.read(buffer); if (i > 0) { digest.update(buffer, 0, i); } - } while(i!=-1); + } while(i != -1); } /** |