Validate file extension for files in application package

Start validating file extension for files application package subdirectories,
only a subset of subdirectories handled for now

8 files changed, 104 insertions, 5 deletions

diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
index 78d195821bf..41896468769 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
@@ -13,12 +13,12 @@ import com.yahoo.config.application.api.ComponentInfo;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.UnparsedConfigDefinition;
 import com.yahoo.config.codegen.DefParser;
+import com.yahoo.config.model.application.AbstractApplicationPackage;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
-import com.yahoo.config.model.application.AbstractApplicationPackage;
 import com.yahoo.io.HexDump;
 import com.yahoo.io.IOUtils;
 import com.yahoo.io.reader.NamedReader;
@@ -33,7 +33,6 @@ import org.w3c.dom.Element;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
-
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.TransformerException;
 import javax.xml.transform.TransformerFactory;
@@ -48,6 +47,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.Reader;
 import java.io.StringReader;
+import java.nio.file.Files;
 import java.security.MessageDigest;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -61,7 +61,6 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.BiFunction;
-import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -69,7 +68,7 @@ import static com.yahoo.text.Lowercase.toLowerCase;
 
 
 /**
- * Application package derived from local files, ie. during deploy.
+ * Application package derived from local files, i.e. during deploy.
  * Construct using {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFile(java.io.File)} or
  * {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFileWithDeployData(java.io.File, DeployData)}.
  *
@@ -749,4 +748,40 @@ public class FilesApplicationPackage extends AbstractApplicationPackage {
         return file;
     }
 
+    /* Validates that files in application dir and subdirectories have a known extension */
+    public void validateFileExtensions(boolean throwIfInvalid) {
+        // TODO: Define this for all subdirs
+        Map<Path, Set<String>> validFileSuffixes = Map.of(
+                QUERY_PROFILES_DIR, Set.of(".xml"),
+                RULES_DIR, Set.of(".sr"),
+                SCHEMAS_DIR, Set.of(".sd", ".expression"),
+                SEARCH_DEFINITIONS_DIR, Set.of(".sd", ".expression"));
+
+        validFileSuffixes.forEach((key, value) -> {
+            java.nio.file.Path path = appDir.toPath().resolve((key.toFile().toPath()));
+            File dir = path.toFile();
+            if ( ! dir.exists() || ! dir.isDirectory()) return;
+
+            try (var filesInPath = Files.list(path)) {
+                filesInPath.forEach(f -> {
+                    validateFileSuffix(path, f, value, throwIfInvalid);
+                });
+            } catch (IOException e) {
+                log.log(Level.WARNING, "Unable to list files in " + dir, e);
+            }
+        });
+    }
+
+    private void validateFileSuffix(java.nio.file.Path dir, java.nio.file.Path pathToFile, Set<String> allowedSuffixes, boolean throwIfInvalid) {
+        String fileName = pathToFile.toFile().getName();
+        if (allowedSuffixes.stream().noneMatch(fileName::endsWith)) {
+            String message = "File in application package with unknown suffix: " +
+                    appDir.toPath().relativize(dir).resolve(fileName) + " Please delete or move file to another directory.";
+            if (throwIfInvalid)
+                throw new IllegalArgumentException(message);
+            else
+                log.log(Level.INFO, message);
+        }
+    }
+
 }
diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java
index fd5437c27de..bea7efb40f4 100644
--- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java
+++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java
@@ -105,7 +105,7 @@ public class FilesApplicationPackageTest {
     }
 
     @Test
-    public void testLegacyOverrides() throws IOException {
+    public void testLegacyOverrides() {
         File appDir = new File("src/test/resources/app-legacy-overrides");
         ApplicationPackage app = FilesApplicationPackage.fromFile(appDir);
         var overrides = app.legacyOverrides();
@@ -143,4 +143,25 @@ public class FilesApplicationPackageTest {
                              .getMessage());
     }
 
+    @Test
+    public void testValidFileExtensions() {
+        File appDir = new File("src/test/resources/app-with-deployment");;
+        FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir);
+        app.validateFileExtensions(true);
+    }
+
+    @Test
+    public void testInvalidFileExtensions() {
+        File appDir = new File("src/test/resources/app-with-invalid-files-in-subdir");;
+        FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir);
+        try {
+            app.validateFileExtensions(true);
+            fail("expected an exception");
+        } catch (IllegalArgumentException e) {
+            assertEquals("File in application package with unknown suffix: search/query-profiles/file-with-invalid.extension " +
+                                 "Please delete or move file to another directory.",
+                         e.getMessage());
+        }
+    }
+
 }
diff --git a/config-application-package/src/test/resources/app-with-deployment/search/query-profiles/default.xml b/config-application-package/src/test/resources/app-with-deployment/search/query-profiles/default.xml
new file mode 100644
index 00000000000..6e3069d8f96
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-deployment/search/query-profiles/default.xml
@@ -0,0 +1,4 @@
+<query-profile id="default">
+    <field name="maxHits">30000</field>
+    <field name="maxOffset">30000</field>
+</query-profile>
diff --git a/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/deployment.xml b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/deployment.xml
new file mode 100644
index 00000000000..3aad0ca6a6a
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/deployment.xml
@@ -0,0 +1,9 @@
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<deployment version='1.0'>
+  <test />
+  <staging />
+  <prod>
+    <region active="true">us-east</region>
+    <region active="false">us-west-1</region>
+  </prod>
+</deployment>
diff --git a/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/hosts.xml b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/hosts.xml
new file mode 100644
index 00000000000..64a07644038
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/hosts.xml
@@ -0,0 +1,10 @@
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts xmlns:deploy="vespa" xmlns:preprocess="properties">
+    <preprocess:properties>
+        <node1.hostname>foo.yahoo.com</node1.hostname>
+        <node1.hostname deploy:environment="dev">bar.yahoo.com</node1.hostname>
+    </preprocess:properties>
+    <host name="${node1.hostname}">
+        <alias>node1</alias>
+    </host>
+</hosts>
diff --git a/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/schemas/music.sd b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/schemas/music.sd
new file mode 100644
index 00000000000..7da7c49c162
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/schemas/music.sd
@@ -0,0 +1,8 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search music {
+    document music {
+        field f type string {
+            indexing: index | summary
+        }
+    }
+}
diff --git a/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/search/query-profiles/file-with-invalid.extension b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/search/query-profiles/file-with-invalid.extension
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/search/query-profiles/file-with-invalid.extension
diff --git a/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/services.xml b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/services.xml
new file mode 100644
index 00000000000..60d08cb615c
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-invalid-files-in-subdir/services.xml
@@ -0,0 +1,12 @@
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services version='1.0'>
+    <admin version='2.0'>
+        <adminserver hostalias='node0'/>
+    </admin>
+    <content version='1.0' id='foo'>
+      <redundancy>1</redundancy>
+      <documents>
+        <document type="music.sd" mode="index" />
+      </documents>
+    </content>
+</services>