Handle .profile files in subdirs of schemas/

12 files changed, 106 insertions, 20 deletions

diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
index 6f94ef5454d..08b500911c3 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
@@ -751,18 +751,24 @@ public class FilesApplicationPackage extends AbstractApplicationPackage {
 
     /* Validates that files in application dir and subdirectories have a known extension */
     public void validateFileExtensions(boolean throwIfInvalid) {
-        validFileExtensions.forEach((key, value) -> {
-            java.nio.file.Path path = appDir.toPath().resolve((key.toFile().toPath()));
-            File subDirectory = path.toFile();
-            if ( ! subDirectory.exists() || ! subDirectory.isDirectory()) return;
-
-            try (var filesInPath = Files.list(path)) {
-                filesInPath.filter(filePath -> filePath.toFile().isFile())
-                           .forEach(f -> validateFileSuffix(path, f, value, throwIfInvalid));
-            } catch (IOException e) {
-                log.log(Level.WARNING, "Unable to list files in " + subDirectory, e);
-            }
-        });
+        validFileExtensions.forEach((subDir, __) -> validateInDir(subDir.toFile().toPath(), throwIfInvalid));
+    }
+
+    private void validateInDir(java.nio.file.Path subDir, boolean throwIfInvalid) {
+        java.nio.file.Path path = appDir.toPath().resolve(subDir);
+        File subDirectory = path.toFile();
+        if ( ! subDirectory.exists() || ! subDirectory.isDirectory()) return;
+
+        try (var filesInPath = Files.list(path)) {
+            filesInPath.forEach(filePath -> {
+                if (filePath.toFile().isDirectory())
+                    validateInDir(appDir.toPath().relativize(filePath), throwIfInvalid);
+                else
+                    validateFileSuffix(filePath, throwIfInvalid);
+            });
+        } catch (IOException e) {
+            log.log(Level.WARNING, "Unable to list files in " + subDirectory, e);
+        }
     }
 
     static {
@@ -777,10 +783,10 @@ public class FilesApplicationPackage extends AbstractApplicationPackage {
                 Map.entry(QUERY_PROFILE_TYPES_DIR, Set.of(".xml")),
                 Map.entry(Path.fromString(ROUTINGTABLES_DIR), Set.of(".xml")),
                 Map.entry(RULES_DIR, Set.of(RULES_NAME_SUFFIX)),
-                // TODO: Might have rank profiles in subdirs: schema-name]/[rank-profile].profile
+                // Note: Might have rank profiles in subdirs: schema-name]/[rank-profile].profile
                 Map.entry(SCHEMAS_DIR, Set.of(SD_NAME_SUFFIX, RANKEXPRESSION_NAME_SUFFIX)),
                 Map.entry(Path.fromString(SEARCHCHAINS_DIR), Set.of(".xml")),
-                // TODO: Might have rank profiles in subdirs: [schema-name]/[rank-profile].profile
+                // Note: Might have rank profiles in subdirs: [schema-name]/[rank-profile].profile
                 Map.entry(SEARCH_DEFINITIONS_DIR, Set.of(SD_NAME_SUFFIX, RANKEXPRESSION_NAME_SUFFIX)),
                 Map.entry(SECURITY_DIR, Set.of(".pem")));
 
@@ -789,11 +795,14 @@ public class FilesApplicationPackage extends AbstractApplicationPackage {
         //   onnx model files (.onnx)
     }
 
-    private void validateFileSuffix(java.nio.file.Path dir, java.nio.file.Path pathToFile, Set<String> allowedSuffixes, boolean throwIfInvalid) {
+    private void validateFileSuffix(java.nio.file.Path pathToFile, boolean throwIfInvalid) {
         String fileName = pathToFile.toFile().getName();
-        if (allowedSuffixes.stream().noneMatch(fileName::endsWith)) {
+        java.nio.file.Path relativeDirectory = appDir.toPath().relativize(pathToFile).getParent();
+        Set<String> allowedExtensions = findAllowedExtensions(relativeDirectory);
+        log.log(Level.FINE, "Checking " + pathToFile + " against " + allowedExtensions);
+        if (allowedExtensions.stream().noneMatch(fileName::endsWith)) {
             String message = "File in application package with unknown suffix: " +
-                    appDir.toPath().relativize(dir).resolve(fileName) + " Please delete or move file to another directory.";
+                    appDir.toPath().relativize(pathToFile.getParent()).resolve(fileName) + ", please delete or move file to another directory.";
             if (throwIfInvalid)
                 throw new IllegalArgumentException(message);
             else
@@ -801,4 +810,24 @@ public class FilesApplicationPackage extends AbstractApplicationPackage {
         }
     }
 
+    private Set<String> findAllowedExtensions(java.nio.file.Path relativeDirectory) {
+        return (isSubDirInSchemas(relativeDirectory))
+                ? Set.of(".profile") // Special case, since subdir in schemas can have any name
+                : validFileExtensions.entrySet().stream()
+                                     .filter(entry -> entry.getKey().equals(Path.fromString(relativeDirectory.toString())))
+                                     .map(Map.Entry::getValue)
+                                     .findFirst()
+                                     .orElse(Set.of());
+    }
+
+    private boolean isSubDirInSchemas(java.nio.file.Path relativeDirectory) {
+        java.nio.file.Path schemasPath = SCHEMAS_DIR.toFile().toPath().getName(0);
+        java.nio.file.Path searchDefinitionsPath = SEARCH_DEFINITIONS_DIR.toFile().toPath().getName(0);
+        if (relativeDirectory.equals(schemasPath) ||relativeDirectory.equals(searchDefinitionsPath)) return false;
+
+        return (relativeDirectory.startsWith(schemasPath + "/")
+                || relativeDirectory.startsWith(SEARCH_DEFINITIONS_DIR.toFile().toPath().getName(0) + "/"));
+    }
+
+
 }
diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java
index bea7efb40f4..e3a27ecddf0 100644
--- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java
+++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java
@@ -10,7 +10,6 @@ import com.yahoo.io.IOUtils;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
@@ -158,8 +157,22 @@ public class FilesApplicationPackageTest {
             app.validateFileExtensions(true);
             fail("expected an exception");
         } catch (IllegalArgumentException e) {
-            assertEquals("File in application package with unknown suffix: search/query-profiles/file-with-invalid.extension " +
-                                 "Please delete or move file to another directory.",
+            assertEquals("File in application package with unknown suffix: search/query-profiles/file-with-invalid.extension, " +
+                                 "please delete or move file to another directory.",
+                         e.getMessage());
+        }
+    }
+
+    @Test
+    public void testInvalidFileExtensionInSubDirOfSubDir() {
+        File appDir = new File("src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/");;
+        FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir);
+        try {
+            app.validateFileExtensions(true);
+            fail("expected an exception");
+        } catch (IllegalArgumentException e) {
+            assertEquals("File in application package with unknown suffix: schemas/foo/bar.junk, " +
+                                 "please delete or move file to another directory.",
                          e.getMessage());
         }
     }
diff --git a/config-application-package/src/test/resources/app-with-deployment/models/a.onnx b/config-application-package/src/test/resources/app-with-deployment/models/a.onnx
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-deployment/models/a.onnx
diff --git a/config-application-package/src/test/resources/app-with-deployment/models/b.tf b/config-application-package/src/test/resources/app-with-deployment/models/b.tf
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-deployment/models/b.tf
diff --git a/config-application-package/src/test/resources/app-with-deployment/models/c.json b/config-application-package/src/test/resources/app-with-deployment/models/c.json
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-deployment/models/c.json
diff --git a/config-application-package/src/test/resources/app-with-deployment/models/d.json.lz4 b/config-application-package/src/test/resources/app-with-deployment/models/d.json.lz4
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-deployment/models/d.json.lz4
diff --git a/config-application-package/src/test/resources/app-with-deployment/schemas/music/default.profile b/config-application-package/src/test/resources/app-with-deployment/schemas/music/default.profile
new file mode 100644
index 00000000000..a0ab4d8225e
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-deployment/schemas/music/default.profile
@@ -0,0 +1,5 @@
+rank-profile default {
+    second-phase {
+        expression: fieldMatch(f)
+    }
+}
diff --git a/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/deployment.xml b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/deployment.xml
new file mode 100644
index 00000000000..3aad0ca6a6a
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/deployment.xml
@@ -0,0 +1,9 @@
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<deployment version='1.0'>
+  <test />
+  <staging />
+  <prod>
+    <region active="true">us-east</region>
+    <region active="false">us-west-1</region>
+  </prod>
+</deployment>
diff --git a/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/hosts.xml b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/hosts.xml
new file mode 100644
index 00000000000..64a07644038
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/hosts.xml
@@ -0,0 +1,10 @@
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts xmlns:deploy="vespa" xmlns:preprocess="properties">
+    <preprocess:properties>
+        <node1.hostname>foo.yahoo.com</node1.hostname>
+        <node1.hostname deploy:environment="dev">bar.yahoo.com</node1.hostname>
+    </preprocess:properties>
+    <host name="${node1.hostname}">
+        <alias>node1</alias>
+    </host>
+</hosts>
diff --git a/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/foo/bar.junk b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/foo/bar.junk
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/foo/bar.junk
diff --git a/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/music.sd b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/music.sd
new file mode 100644
index 00000000000..7da7c49c162
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/schemas/music.sd
@@ -0,0 +1,8 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search music {
+    document music {
+        field f type string {
+            indexing: index | summary
+        }
+    }
+}
diff --git a/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/services.xml b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/services.xml
new file mode 100644
index 00000000000..60d08cb615c
--- /dev/null
+++ b/config-application-package/src/test/resources/app-with-files-with-invalid-extension-in-subdir-of-subdir/services.xml
@@ -0,0 +1,12 @@
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services version='1.0'>
+    <admin version='2.0'>
+        <adminserver hostalias='node0'/>
+    </admin>
+    <content version='1.0' id='foo'>
+      <redundancy>1</redundancy>
+      <documents>
+        <document type="music.sd" mode="index" />
+      </documents>
+    </content>
+</services>