diff options
author | Harald Musum <musum@yahooinc.com> | 2022-04-28 15:01:42 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2022-04-28 15:01:42 +0200 |
commit | d5a2caa23b848a18352127b502f4817aa9bfc45d (patch) | |
tree | 1cd48b848b802bc3590d681a83c9b77cf33b7c86 /config-application-package | |
parent | 04add1191a91616173e3a84a9e69a9215a108d6d (diff) |
Validate file extension for app files, part 4
* Allow .gitignore files everywhere and .profile files in schemas/ directory
* Move exception handling and logging to config server code
Diffstat (limited to 'config-application-package')
3 files changed, 38 insertions, 35 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 08b500911c3..793020ce104 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -750,11 +751,11 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { } /* Validates that files in application dir and subdirectories have a known extension */ - public void validateFileExtensions(boolean throwIfInvalid) { - validFileExtensions.forEach((subDir, __) -> validateInDir(subDir.toFile().toPath(), throwIfInvalid)); + public void validateFileExtensions() { + validFileExtensions.forEach((subDir, __) -> validateInDir(subDir.toFile().toPath())); } - private void validateInDir(java.nio.file.Path subDir, boolean throwIfInvalid) { + private void validateInDir(java.nio.file.Path subDir) { java.nio.file.Path path = appDir.toPath().resolve(subDir); File subDirectory = path.toFile(); if ( ! subDirectory.exists() || ! subDirectory.isDirectory()) return; @@ -762,9 +763,9 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { try (var filesInPath = Files.list(path)) { filesInPath.forEach(filePath -> { if (filePath.toFile().isDirectory()) - validateInDir(appDir.toPath().relativize(filePath), throwIfInvalid); + validateInDir(appDir.toPath().relativize(filePath)); else - validateFileSuffix(filePath, throwIfInvalid); + validateFileExtensions(filePath); }); } catch (IOException e) { log.log(Level.WARNING, "Unable to list files in " + subDirectory, e); @@ -772,11 +773,15 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { } static { + // Note: Directories intentionally not validated: MODELS_DIR (custom models can contain files with any extension) + + // TODO: Files that according to doc (https://docs.vespa.ai/en/reference/schema-reference.html) can be anywhere in the application package: + // constant tensors (.json, .json.lz4) + // onnx model files (.onnx) validFileExtensions = Map.ofEntries( Map.entry(Path.fromString(COMPONENT_DIR), Set.of(".jar")), Map.entry(CONSTANTS_DIR, Set.of(".json", ".json.lz4")), Map.entry(Path.fromString(DOCPROCCHAINS_DIR), Set.of(".xml")), - // Map.entry(MODELS_DIR, Set.of(".model")), TODO: Enable on Vespa 8 Map.entry(PAGE_TEMPLATES_DIR, Set.of(".xml")), Map.entry(Path.fromString(PROCESSORCHAINS_DIR), Set.of(".xml")), Map.entry(QUERY_PROFILES_DIR, Set.of(".xml")), @@ -789,35 +794,34 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { // Note: Might have rank profiles in subdirs: [schema-name]/[rank-profile].profile Map.entry(SEARCH_DEFINITIONS_DIR, Set.of(SD_NAME_SUFFIX, RANKEXPRESSION_NAME_SUFFIX)), Map.entry(SECURITY_DIR, Set.of(".pem"))); - - // TODO: Files that according to doc (https://docs.vespa.ai/en/reference/schema-reference.html) can be anywhere in the application package: - // constant tensors (.json, .json.lz4) - // onnx model files (.onnx) } - private void validateFileSuffix(java.nio.file.Path pathToFile, boolean throwIfInvalid) { - String fileName = pathToFile.toFile().getName(); - java.nio.file.Path relativeDirectory = appDir.toPath().relativize(pathToFile).getParent(); - Set<String> allowedExtensions = findAllowedExtensions(relativeDirectory); + private void validateFileExtensions(java.nio.file.Path pathToFile) { + Set<String> allowedExtensions = findAllowedExtensions(appDir.toPath().relativize(pathToFile).getParent()); log.log(Level.FINE, "Checking " + pathToFile + " against " + allowedExtensions); + String fileName = pathToFile.toFile().getName(); if (allowedExtensions.stream().noneMatch(fileName::endsWith)) { - String message = "File in application package with unknown suffix: " + + String message = "File in application package with unknown extension: " + appDir.toPath().relativize(pathToFile.getParent()).resolve(fileName) + ", please delete or move file to another directory."; - if (throwIfInvalid) - throw new IllegalArgumentException(message); - else - log.log(Level.INFO, message); + throw new IllegalArgumentException(message); } } private Set<String> findAllowedExtensions(java.nio.file.Path relativeDirectory) { - return (isSubDirInSchemas(relativeDirectory)) - ? Set.of(".profile") // Special case, since subdir in schemas can have any name - : validFileExtensions.entrySet().stream() - .filter(entry -> entry.getKey().equals(Path.fromString(relativeDirectory.toString()))) - .map(Map.Entry::getValue) - .findFirst() - .orElse(Set.of()); + Set<String> validExtensions = new HashSet<>(); + validExtensions.add(".gitignore"); + + // Special case, since subdirs in schemas/ can have any name + if (isSubDirInSchemas(relativeDirectory)) + validExtensions.add(".profile"); + else + validExtensions.addAll(validFileExtensions.entrySet().stream() + .filter(entry -> entry.getKey() + .equals(Path.fromString(relativeDirectory.toString()))) + .map(Map.Entry::getValue) + .findFirst() + .orElse(Set.of())); + return validExtensions; } private boolean isSubDirInSchemas(java.nio.file.Path relativeDirectory) { @@ -829,5 +833,4 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { || relativeDirectory.startsWith(SEARCH_DEFINITIONS_DIR.toFile().toPath().getName(0) + "/")); } - } diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index e3a27ecddf0..29bc2b6fbd6 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -144,20 +144,20 @@ public class FilesApplicationPackageTest { @Test public void testValidFileExtensions() { - File appDir = new File("src/test/resources/app-with-deployment");; + File appDir = new File("src/test/resources/app-with-deployment"); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); - app.validateFileExtensions(true); + app.validateFileExtensions(); } @Test public void testInvalidFileExtensions() { - File appDir = new File("src/test/resources/app-with-invalid-files-in-subdir");; + File appDir = new File("src/test/resources/app-with-invalid-files-in-subdir"); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); try { - app.validateFileExtensions(true); + app.validateFileExtensions(); fail("expected an exception"); } catch (IllegalArgumentException e) { - assertEquals("File in application package with unknown suffix: search/query-profiles/file-with-invalid.extension, " + + assertEquals("File in application package with unknown extension: search/query-profiles/file-with-invalid.extension, " + "please delete or move file to another directory.", e.getMessage()); } @@ -165,13 +165,13 @@ public class FilesApplicationPackageTest { @Test public void testInvalidFileExtensionInSubDirOfSubDir() { - File appDir = new File("src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/");; + File appDir = new File("src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/"); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); try { - app.validateFileExtensions(true); + app.validateFileExtensions(); fail("expected an exception"); } catch (IllegalArgumentException e) { - assertEquals("File in application package with unknown suffix: schemas/foo/bar.junk, " + + assertEquals("File in application package with unknown extension: schemas/foo/bar.junk, " + "please delete or move file to another directory.", e.getMessage()); } diff --git a/config-application-package/src/test/resources/app-with-deployment/schemas/.gitignore b/config-application-package/src/test/resources/app-with-deployment/schemas/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config-application-package/src/test/resources/app-with-deployment/schemas/.gitignore |