diff options
author | Harald Musum <musum@yahooinc.com> | 2022-04-28 11:06:58 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2022-04-28 11:06:58 +0200 |
commit | fe69ccd756fdd4c32be23b4f043b93ecbbd23603 (patch) | |
tree | 31492180a3ebdb7652240f13963382479d6cbd12 /config-application-package | |
parent | 03f1ea79f5a6ad8d32eaf03b77172e04eab1e0d7 (diff) |
Handle .profile files in subdirs of schemas/
Diffstat (limited to 'config-application-package')
12 files changed, 106 insertions, 20 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 6f94ef5454d..08b500911c3 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -751,18 +751,24 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { /* Validates that files in application dir and subdirectories have a known extension */ public void validateFileExtensions(boolean throwIfInvalid) { - validFileExtensions.forEach((key, value) -> { - java.nio.file.Path path = appDir.toPath().resolve((key.toFile().toPath())); - File subDirectory = path.toFile(); - if ( ! subDirectory.exists() || ! subDirectory.isDirectory()) return; - - try (var filesInPath = Files.list(path)) { - filesInPath.filter(filePath -> filePath.toFile().isFile()) - .forEach(f -> validateFileSuffix(path, f, value, throwIfInvalid)); - } catch (IOException e) { - log.log(Level.WARNING, "Unable to list files in " + subDirectory, e); - } - }); + validFileExtensions.forEach((subDir, __) -> validateInDir(subDir.toFile().toPath(), throwIfInvalid)); + } + + private void validateInDir(java.nio.file.Path subDir, boolean throwIfInvalid) { + java.nio.file.Path path = appDir.toPath().resolve(subDir); + File subDirectory = path.toFile(); + if ( ! subDirectory.exists() || ! subDirectory.isDirectory()) return; + + try (var filesInPath = Files.list(path)) { + filesInPath.forEach(filePath -> { + if (filePath.toFile().isDirectory()) + validateInDir(appDir.toPath().relativize(filePath), throwIfInvalid); + else + validateFileSuffix(filePath, throwIfInvalid); + }); + } catch (IOException e) { + log.log(Level.WARNING, "Unable to list files in " + subDirectory, e); + } } static { @@ -777,10 +783,10 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { Map.entry(QUERY_PROFILE_TYPES_DIR, Set.of(".xml")), Map.entry(Path.fromString(ROUTINGTABLES_DIR), Set.of(".xml")), Map.entry(RULES_DIR, Set.of(RULES_NAME_SUFFIX)), - // TODO: Might have rank profiles in subdirs: schema-name]/[rank-profile].profile + // Note: Might have rank profiles in subdirs: schema-name]/[rank-profile].profile Map.entry(SCHEMAS_DIR, Set.of(SD_NAME_SUFFIX, RANKEXPRESSION_NAME_SUFFIX)), Map.entry(Path.fromString(SEARCHCHAINS_DIR), Set.of(".xml")), - // TODO: Might have rank profiles in subdirs: [schema-name]/[rank-profile].profile + // Note: Might have rank profiles in subdirs: [schema-name]/[rank-profile].profile Map.entry(SEARCH_DEFINITIONS_DIR, Set.of(SD_NAME_SUFFIX, RANKEXPRESSION_NAME_SUFFIX)), Map.entry(SECURITY_DIR, Set.of(".pem"))); @@ -789,11 +795,14 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { // onnx model files (.onnx) } - private void validateFileSuffix(java.nio.file.Path dir, java.nio.file.Path pathToFile, Set<String> allowedSuffixes, boolean throwIfInvalid) { + private void validateFileSuffix(java.nio.file.Path pathToFile, boolean throwIfInvalid) { String fileName = pathToFile.toFile().getName(); - if (allowedSuffixes.stream().noneMatch(fileName::endsWith)) { + java.nio.file.Path relativeDirectory = appDir.toPath().relativize(pathToFile).getParent(); + Set<String> allowedExtensions = findAllowedExtensions(relativeDirectory); + log.log(Level.FINE, "Checking " + pathToFile + " against " + allowedExtensions); + if (allowedExtensions.stream().noneMatch(fileName::endsWith)) { String message = "File in application package with unknown suffix: " + - appDir.toPath().relativize(dir).resolve(fileName) + " Please delete or move file to another directory."; + appDir.toPath().relativize(pathToFile.getParent()).resolve(fileName) + ", please delete or move file to another directory."; if (throwIfInvalid) throw new IllegalArgumentException(message); else @@ -801,4 +810,24 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { } } + private Set<String> findAllowedExtensions(java.nio.file.Path relativeDirectory) { + return (isSubDirInSchemas(relativeDirectory)) + ? Set.of(".profile") // Special case, since subdir in schemas can have any name + : validFileExtensions.entrySet().stream() + .filter(entry -> entry.getKey().equals(Path.fromString(relativeDirectory.toString()))) + .map(Map.Entry::getValue) + .findFirst() + .orElse(Set.of()); + } + + private boolean isSubDirInSchemas(java.nio.file.Path relativeDirectory) { + java.nio.file.Path schemasPath = SCHEMAS_DIR.toFile().toPath().getName(0); + java.nio.file.Path searchDefinitionsPath = SEARCH_DEFINITIONS_DIR.toFile().toPath().getName(0); + if (relativeDirectory.equals(schemasPath) ||relativeDirectory.equals(searchDefinitionsPath)) return false; + + return (relativeDirectory.startsWith(schemasPath + "/") + || relativeDirectory.startsWith(SEARCH_DEFINITIONS_DIR.toFile().toPath().getName(0) + "/")); + } + + } diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index bea7efb40f4..e3a27ecddf0 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -10,7 +10,6 @@ import com.yahoo.io.IOUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.FileReader; import java.io.IOException; @@ -158,8 +157,22 @@ public class FilesApplicationPackageTest { app.validateFileExtensions(true); fail("expected an exception"); } catch (IllegalArgumentException e) { - assertEquals("File in application package with unknown suffix: search/query-profiles/file-with-invalid.extension " + - "Please delete or move file to another directory.", + assertEquals("File in application package with unknown suffix: search/query-profiles/file-with-invalid.extension, " + + "please delete or move file to another directory.", + e.getMessage()); + } + } + + @Test + public void testInvalidFileExtensionInSubDirOfSubDir() { + File appDir = new File("src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/");; + FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); + try { + app.validateFileExtensions(true); + fail("expected an exception"); + } catch (IllegalArgumentException e) { + assertEquals("File in application package with unknown suffix: schemas/foo/bar.junk, " + + "please delete or move file to another directory.", e.getMessage()); } } diff --git a/config-application-package/src/test/resources/app-with-deployment/models/a.onnx b/config-application-package/src/test/resources/app-with-deployment/models/a.onnx new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config-application-package/src/test/resources/app-with-deployment/models/a.onnx diff --git a/config-application-package/src/test/resources/app-with-deployment/models/b.tf b/config-application-package/src/test/resources/app-with-deployment/models/b.tf new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config-application-package/src/test/resources/app-with-deployment/models/b.tf diff --git a/config-application-package/src/test/resources/app-with-deployment/models/c.json b/config-application-package/src/test/resources/app-with-deployment/models/c.json new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config-application-package/src/test/resources/app-with-deployment/models/c.json diff --git a/config-application-package/src/test/resources/app-with-deployment/models/d.json.lz4 b/config-application-package/src/test/resources/app-with-deployment/models/d.json.lz4 new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config-application-package/src/test/resources/app-with-deployment/models/d.json.lz4 diff --git a/config-application-package/src/test/resources/app-with-deployment/schemas/music/default.profile b/config-application-package/src/test/resources/app-with-deployment/schemas/music/default.profile new file mode 100644 index 00000000000..a0ab4d8225e --- /dev/null +++ b/config-application-package/src/test/resources/app-with-deployment/schemas/music/default.profile @@ -0,0 +1,5 @@ +rank-profile default { + second-phase { + expression: fieldMatch(f) + } +} diff --git a/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/deployment.xml b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/deployment.xml new file mode 100644 index 00000000000..3aad0ca6a6a --- /dev/null +++ b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/deployment.xml @@ -0,0 +1,9 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<deployment version='1.0'> + <test /> + <staging /> + <prod> + <region active="true">us-east</region> + <region active="false">us-west-1</region> + </prod> +</deployment> diff --git a/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/hosts.xml b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/hosts.xml new file mode 100644 index 00000000000..64a07644038 --- /dev/null +++ b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/hosts.xml @@ -0,0 +1,10 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts xmlns:deploy="vespa" xmlns:preprocess="properties"> + <preprocess:properties> + <node1.hostname>foo.yahoo.com</node1.hostname> + <node1.hostname deploy:environment="dev">bar.yahoo.com</node1.hostname> + </preprocess:properties> + <host name="${node1.hostname}"> + <alias>node1</alias> + </host> +</hosts> diff --git a/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/foo/bar.junk b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/foo/bar.junk new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/foo/bar.junk diff --git a/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/music.sd b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/music.sd new file mode 100644 index 00000000000..7da7c49c162 --- /dev/null +++ b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/music.sd @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search music { + document music { + field f type string { + indexing: index | summary + } + } +} diff --git a/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/services.xml b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/services.xml new file mode 100644 index 00000000000..60d08cb615c --- /dev/null +++ b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/services.xml @@ -0,0 +1,12 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version='1.0'> + <admin version='2.0'> + <adminserver hostalias='node0'/> + </admin> + <content version='1.0' id='foo'> + <redundancy>1</redundancy> + <documents> + <document type="music.sd" mode="index" /> + </documents> + </content> +</services> |