From 90402e33fe73f74e38f77d6a1ddc5b892aaf03f2 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 27 Apr 2022 09:44:54 +0200 Subject: Validate file extension for files in application package Start validating file extension for files application package subdirectories, only a subset of subdirectories handled for now --- .../provider/FilesApplicationPackage.java | 43 ++++++++++++++++++++-- .../provider/FilesApplicationPackageTest.java | 23 +++++++++++- .../search/query-profiles/default.xml | 4 ++ .../deployment.xml | 9 +++++ .../app-with-invalid-files-in-subdir/hosts.xml | 10 +++++ .../schemas/music.sd | 8 ++++ .../query-profiles/file-with-invalid.extension | 0 .../app-with-invalid-files-in-subdir/services.xml | 12 ++++++ 8 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 config-application-package/src/test/resources/app-with-deployment/search/query-profiles/default.xml create mode 100644 config-application-package/src/test/resources/app-with-invalid-files-in-subdir/deployment.xml create mode 100644 config-application-package/src/test/resources/app-with-invalid-files-in-subdir/hosts.xml create mode 100644 config-application-package/src/test/resources/app-with-invalid-files-in-subdir/schemas/music.sd create mode 100644 config-application-package/src/test/resources/app-with-invalid-files-in-subdir/search/query-profiles/file-with-invalid.extension create mode 100644 config-application-package/src/test/resources/app-with-invalid-files-in-subdir/services.xml (limited to 'config-application-package/src') diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 78d195821bf..41896468769 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -13,12 +13,12 @@ import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.model.application.AbstractApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.config.model.application.AbstractApplicationPackage; import com.yahoo.io.HexDump; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; @@ -33,7 +33,6 @@ import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; - import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; @@ -48,6 +47,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.Reader; import java.io.StringReader; +import java.nio.file.Files; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; @@ -61,7 +61,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.BiFunction; -import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; @@ -69,7 +68,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; /** - * Application package derived from local files, ie. during deploy. + * Application package derived from local files, i.e. during deploy. * Construct using {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFile(java.io.File)} or * {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFileWithDeployData(java.io.File, DeployData)}. * @@ -749,4 +748,40 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { return file; } + /* Validates that files in application dir and subdirectories have a known extension */ + public void validateFileExtensions(boolean throwIfInvalid) { + // TODO: Define this for all subdirs + Map> validFileSuffixes = Map.of( + QUERY_PROFILES_DIR, Set.of(".xml"), + RULES_DIR, Set.of(".sr"), + SCHEMAS_DIR, Set.of(".sd", ".expression"), + SEARCH_DEFINITIONS_DIR, Set.of(".sd", ".expression")); + + validFileSuffixes.forEach((key, value) -> { + java.nio.file.Path path = appDir.toPath().resolve((key.toFile().toPath())); + File dir = path.toFile(); + if ( ! dir.exists() || ! dir.isDirectory()) return; + + try (var filesInPath = Files.list(path)) { + filesInPath.forEach(f -> { + validateFileSuffix(path, f, value, throwIfInvalid); + }); + } catch (IOException e) { + log.log(Level.WARNING, "Unable to list files in " + dir, e); + } + }); + } + + private void validateFileSuffix(java.nio.file.Path dir, java.nio.file.Path pathToFile, Set allowedSuffixes, boolean throwIfInvalid) { + String fileName = pathToFile.toFile().getName(); + if (allowedSuffixes.stream().noneMatch(fileName::endsWith)) { + String message = "File in application package with unknown suffix: " + + appDir.toPath().relativize(dir).resolve(fileName) + " Please delete or move file to another directory."; + if (throwIfInvalid) + throw new IllegalArgumentException(message); + else + log.log(Level.INFO, message); + } + } + } diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index fd5437c27de..bea7efb40f4 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -105,7 +105,7 @@ public class FilesApplicationPackageTest { } @Test - public void testLegacyOverrides() throws IOException { + public void testLegacyOverrides() { File appDir = new File("src/test/resources/app-legacy-overrides"); ApplicationPackage app = FilesApplicationPackage.fromFile(appDir); var overrides = app.legacyOverrides(); @@ -143,4 +143,25 @@ public class FilesApplicationPackageTest { .getMessage()); } + @Test + public void testValidFileExtensions() { + File appDir = new File("src/test/resources/app-with-deployment");; + FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); + app.validateFileExtensions(true); + } + + @Test + public void testInvalidFileExtensions() { + File appDir = new File("src/test/resources/app-with-invalid-files-in-subdir");; + FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); + try { + app.validateFileExtensions(true); + fail("expected an exception"); + } catch (IllegalArgumentException e) { + assertEquals("File in application package with unknown suffix: search/query-profiles/file-with-invalid.extension " + + "Please delete or move file to another directory.", + e.getMessage()); + } + } + } diff --git a/config-application-package/src/test/resources/app-with-deployment/search/query-profiles/default.xml b/config-application-package/src/test/resources/app-with-deployment/search/query-profiles/default.xml new file mode 100644 index 00000000000..6e3069d8f96 --- /dev/null +++ b/config-application-package/src/test/resources/app-with-deployment/search/query-profiles/default.xml @@ -0,0 +1,4 @@ + + 30000 + 30000 + diff --git a/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/deployment.xml b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/deployment.xml new file mode 100644 index 00000000000..3aad0ca6a6a --- /dev/null +++ b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/deployment.xml @@ -0,0 +1,9 @@ + + + + + + us-east + us-west-1 + + diff --git a/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/hosts.xml b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/hosts.xml new file mode 100644 index 00000000000..64a07644038 --- /dev/null +++ b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/hosts.xml @@ -0,0 +1,10 @@ + + + + foo.yahoo.com + bar.yahoo.com + + + node1 + + diff --git a/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/schemas/music.sd b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/schemas/music.sd new file mode 100644 index 00000000000..7da7c49c162 --- /dev/null +++ b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/schemas/music.sd @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search music { + document music { + field f type string { + indexing: index | summary + } + } +} diff --git a/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/search/query-profiles/file-with-invalid.extension b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/search/query-profiles/file-with-invalid.extension new file mode 100644 index 00000000000..e69de29bb2d diff --git a/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/services.xml b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/services.xml new file mode 100644 index 00000000000..60d08cb615c --- /dev/null +++ b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/services.xml @@ -0,0 +1,12 @@ + + + + + + + 1 + + + + + -- cgit v1.2.3