Look up modular profiles

10 files changed, 336 insertions, 57 deletions

diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java
index c8110bbd977..cc214f7320c 100644
--- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java
@@ -183,7 +183,7 @@ public class SdUtil {
                                                              .filter(f -> f.getName().equals(functionName))
                                                              .findAny();
         if (function.isPresent()) return function;
-        for (var parent : new RankProfile(profile).findInherited()) {
+        for (var parent : new RankProfile(profile, null).findInherited()) {
             function = findFunction(functionName, parent.definition());
             if (function.isPresent()) return function;
         }
diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdHierarchyUtil.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdHierarchyUtil.java
index a3e25cea745..9abd9511c84 100644
--- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdHierarchyUtil.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdHierarchyUtil.java
@@ -43,7 +43,7 @@ public class SdHierarchyUtil {
 
     private static boolean isChildOf(SdRankProfileDefinition targetProfile, SdRankProfileDefinition thisProfile) {
         if (thisProfile.getName().equals(targetProfile.getName())) return true;
-        return new RankProfile(thisProfile).findInherited()
+        return new RankProfile(thisProfile, null).findInherited()
                                            .stream()
                                            .anyMatch(parent -> isChildOf(targetProfile, parent.definition()));
     }
diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/RankProfile.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/RankProfile.java
index 2cc299a4eb3..3b559cecd27 100644
--- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/RankProfile.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/RankProfile.java
@@ -5,13 +5,10 @@ import ai.vespa.intellij.schema.SdUtil;
 import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
 import ai.vespa.intellij.schema.psi.SdTypes;
 import com.intellij.lang.ASTNode;
-import com.intellij.openapi.project.Project;
-import com.intellij.psi.PsiElement;
-import com.intellij.psi.util.PsiTreeUtil;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Optional;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 /**
@@ -23,8 +20,11 @@ public class RankProfile {
 
     private final SdRankProfileDefinition definition;
 
-    public RankProfile(SdRankProfileDefinition definition) {
-        this.definition = definition;
+    private final Schema owner;
+
+    public RankProfile(SdRankProfileDefinition definition, Schema owner) {
+        this.definition = Objects.requireNonNull(definition);
+        this.owner = owner;
     }
 
     public String name() { return definition.getName(); }
@@ -42,7 +42,8 @@ public class RankProfile {
         return inherits().stream()
                          .map(parentIdentifierAST -> parentIdentifierAST.getPsi().getReference())
                          .filter(reference -> reference != null)
-                         .map(reference -> new RankProfile((SdRankProfileDefinition)reference.resolve()))
+                         .map(reference -> owner.rankProfile(reference.getCanonicalText()))
+                         .flatMap(r -> r.stream())
                          .collect(Collectors.toList());
     }
 
@@ -65,21 +66,4 @@ public class RankProfile {
         return inherited;
     }
 
-    /**
-     * Returns the profile of the given name from the given file.
-     *
-     * @throws IllegalArgumentException if not found
-     */
-    public static RankProfile fromProjectFile(Project project, String filePath, String profileName) {
-        PsiElement root = Schema.load(project, filePath);
-        Optional<SdRankProfileDefinition> definition =
-                PsiTreeUtil.collectElementsOfType(root, SdRankProfileDefinition.class)
-                   .stream()
-                   .filter(p -> p.getName().equals(profileName))
-                   .findAny();
-        if (definition.isEmpty())
-            throw new IllegalArgumentException("Rank profile '" + profileName + "' is not present in " + filePath);
-        return new RankProfile(definition.get());
-    }
-
 }
diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/Schema.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/Schema.java
index 3b67a5a0290..3670d095be2 100644
--- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/Schema.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/Schema.java
@@ -3,11 +3,11 @@ package ai.vespa.intellij.schema.model;
 
 import ai.vespa.intellij.schema.psi.SdFile;
 import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
-import ai.vespa.intellij.schema.psi.SdSchemaBody;
-import ai.vespa.intellij.schema.psi.SdSchemaDefinition;
+import ai.vespa.intellij.schema.utils.Files;
+import ai.vespa.intellij.schema.utils.Path;
 import com.intellij.openapi.project.Project;
-import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiFile;
+import com.intellij.psi.PsiFileSystemItem;
 import com.intellij.psi.search.FilenameIndex;
 import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.psi.util.PsiTreeUtil;
@@ -23,30 +23,52 @@ public class Schema {
 
     private final SdFile definition;
 
-    public Schema(SdFile definition) {
+    /** The path to this schema */
+    private final Path path;
+
+    /** The project this is part of */
+    private final Project project;
+
+    public Schema(SdFile definition, Path path, Project project) {
         this.definition = definition;
+        this.path = path;
+        this.project = project;
     }
 
+    public String name() { return definition.getName().substring(0, definition.getName().length() - 3); }
+
     public SdFile definition() { return definition; }
 
+    /** Returns a rank profile belonging to this, defined either inside it or in a separate .profile file */
+    public Optional<RankProfile> rankProfile(String name) {
+        var definition = findProfileElement(name, this.definition);
+        if (definition.isEmpty()) { // Defined in a separate file schema-name/profile-name.profile
+            Optional<PsiFile> file = Files.open(path.getParentPath().append(name()).append(name + ".profile"), project);
+            if (file.isPresent())
+                definition = findProfileElement(name, file.get());
+        }
+        return definition.map(d -> new RankProfile(d, this));
+    }
+
+    private Optional<SdRankProfileDefinition> findProfileElement(String name, PsiFile file) {
+        return PsiTreeUtil.collectElementsOfType(file, SdRankProfileDefinition.class)
+                          .stream()
+                          .filter(p -> name.equals(p.getName()))
+                          .findAny();
+    }
+
     /**
      * Returns the profile of the given name from the given file.
      *
      * @throws IllegalArgumentException if not found
      */
-    public static Schema fromProjectFile(Project project, String filePath) {
-        return new Schema((SdFile)load(project, filePath));
-    }
-
-    static PsiElement load(Project project, String filePath) {
-        PsiFile[] psiFile = FilenameIndex.getFilesByName(project, filePath, GlobalSearchScope.allScope(project));
-        if (psiFile.length == 0)
-            throw new IllegalArgumentException(filePath + " could not be opened");
-        if (psiFile.length > 1)
-            throw new IllegalArgumentException("Multiple files matches " + filePath);
-        if ( ! (psiFile[0] instanceof SdFile))
-            throw new IllegalArgumentException(filePath + " is not a schema file");
-        return psiFile[0];
+    public static Schema fromProjectFile(Project project, Path file) {
+        Optional<PsiFile> psiFile = Files.open(file, project);
+        if (psiFile.isEmpty())
+            throw new IllegalArgumentException("Could not find file '" + file + "'");
+        if ( ! (psiFile.get() instanceof SdFile))
+            throw new IllegalArgumentException(file + " is not a schema file");
+        return new Schema((SdFile)psiFile.get(), file, project);
     }
 
 }
diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java
index fae8d7aa7e4..7a7df9913dd 100644
--- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java
@@ -25,7 +25,7 @@ public class SdFile extends PsiFileBase {
     
     @Override
     public String toString() {
-        return "Sd File";
+        return "Sd file '" + getName() + "'";
     }
 
 }
diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFunctionDefinitionInterface.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFunctionDefinitionInterface.java
index 48eb9b3831c..189c02774c2 100644
--- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFunctionDefinitionInterface.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFunctionDefinitionInterface.java
@@ -15,7 +15,7 @@ public interface SdFunctionDefinitionInterface extends SdDeclaration {
         String functionName = this.getName();
         SdRankProfileDefinition thisRankProfile = PsiTreeUtil.getParentOfType(this, SdRankProfileDefinition.class);
         if (thisRankProfile == null) return false;
-        for (var parentProfile : new RankProfile(thisRankProfile).findInherited()) {
+        for (var parentProfile : new RankProfile(thisRankProfile, null).findInherited()) {
             if (containsFunction(functionName, parentProfile.definition()))
                 return true;
         }
@@ -23,9 +23,9 @@ public interface SdFunctionDefinitionInterface extends SdDeclaration {
     }
 
     default boolean containsFunction(String functionName, SdRankProfileDefinition rankProfile) {
-        if (SdUtil.functionsIn(new RankProfile(rankProfile)).containsKey(functionName))
+        if (SdUtil.functionsIn(new RankProfile(rankProfile, null)).containsKey(functionName))
             return true;
-        for (var parentProfile : new RankProfile(rankProfile).findInherited()) {
+        for (var parentProfile : new RankProfile(rankProfile, null).findInherited()) {
             if (containsFunction(functionName, parentProfile.definition()))
                 return true;
         }
diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/utils/Files.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/utils/Files.java
new file mode 100644
index 00000000000..7ac88a9650d
--- /dev/null
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/utils/Files.java
@@ -0,0 +1,36 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.intellij.schema.utils;
+
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.PsiFile;
+import com.intellij.psi.PsiManager;
+import com.intellij.psi.search.FilenameIndex;
+import com.intellij.psi.search.GlobalSearchScope;
+
+import java.util.Collection;
+import java.util.Optional;
+
+/**
+ * Provide workable file operations on top of IntelliJ's API.
+ */
+public class Files {
+
+    /**
+     * Opens a file.
+     *
+     * @return the file or empty if not found
+     */
+    public static Optional<PsiFile> open(Path file, Project project) {
+        // Apparently there s no API for getting a file by path.
+        // This method returns all files having a particular name.
+        for (VirtualFile candidate : FilenameIndex.getVirtualFilesByName(file.last(),
+                                                                         GlobalSearchScope.allScope(project))) {
+            // Not safe, but (at least in tests) there doesn't appear to be a way to get the workspace root (/src)
+            if (candidate.getPath().endsWith(file.getRelative()))
+                return Optional.of(PsiManager.getInstance(project).findFile(candidate));
+        }
+        return Optional.empty();
+    }
+
+}
diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/utils/Path.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/utils/Path.java
new file mode 100644
index 00000000000..07b2e7f3adf
--- /dev/null
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/utils/Path.java
@@ -0,0 +1,218 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.intellij.schema.utils;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Represents a path represented by a list of elements. Immutable
+ *
+ * @author Ulf Lilleengen
+ * @author bratseth
+ */
+public final class Path {
+
+    private final String delimiter;
+    private final List<String> elements;
+
+    /** Creates an empty path */
+    private Path(String delimiter) {
+        this(new ArrayList<>(), delimiter);
+    }
+
+    /**
+     * Create a new path as a copy of the provided path
+     *
+     * @param path the path to copy
+     */
+    private Path(Path path) {
+        this(path.elements, path.delimiter);
+    }
+
+    /**
+     * Create path with given elements
+     *
+     * @param elements a list of path elements
+     */
+    private Path(List<String> elements, String delimiter) {
+        this.elements = List.copyOf(elements);
+        this.delimiter = delimiter;
+    }
+
+    /** Returns whether this path is an immediate child of the given path */
+    public boolean isChildOf(Path parent) {
+        return toString().startsWith(parent.toString()) && this.elements.size() -1 == parent.elements.size();
+    }
+
+    /**
+     * Add path elements by splitting based on delimiter and appending to elements.
+     */
+    private static List<String> elementsOf(String path, String delimiter) {
+        return Arrays.stream(path.split(delimiter)).filter(e -> !"".equals(e)).collect(Collectors.toList());
+    }
+
+    /**
+     * Append an element to the path. Returns a new path with the given path appended.
+     *
+     * @param path the path to append to this
+     * @return the new path
+     */
+    public Path append(String path) {
+        List<String> newElements = new ArrayList<>(this.elements);
+        newElements.addAll(elementsOf(path, delimiter));
+        return new Path(newElements, delimiter);
+    }
+
+    /**
+     * Appends a path to another path, thereby creating a new path with the provided path
+     * appended to this.
+     *
+     * @param path the path to append
+     * @return a new path with argument appended to it
+     */
+    public Path append(Path path) {
+        List<String> newElements = new ArrayList<>(this.elements);
+        newElements.addAll(path.elements());
+        return new Path(newElements, delimiter);
+    }
+
+    /** Returns the name of this path element, typically the last element in the path string */
+    public String getName() {
+        if (elements.isEmpty()) return "";
+        return elements.get(elements.size() - 1);
+    }
+
+    /** Returns a string representation of the path represented by this */
+    public String getRelative() {
+        if (elements.isEmpty()) {
+            return "";
+        }
+        StringBuilder sb = new StringBuilder();
+        sb.append(elements.get(0));
+        for (int i = 1; i < elements.size(); i++) {
+            sb.append(delimiter);
+            sb.append(elements.get(i));
+        }
+        return sb.toString();
+    }
+
+    /** Returns the parent path: A path containing all elements of this except the last */
+    public Path getParentPath() {
+        ArrayList<String> parentElements = new ArrayList<>();
+        if (elements.size() > 1) {
+            for (int i = 0; i < elements.size() - 1; i++) {
+                parentElements.add(elements.get(i));
+            }
+        }
+        return new Path(parentElements, delimiter);
+    }
+
+    /** Returns the child path: A path containing all elements of this except the first */
+    public Path getChildPath() {
+        ArrayList<String> childElements = new ArrayList<>();
+        if (elements.size() > 1) {
+            for (int i = 1; i < elements.size(); i++) {
+                childElements.add(elements.get(i));
+            }
+        }
+        return new Path(childElements, delimiter);
+    }
+
+    /** Returns the last element in this, or the empty string if this path is empty */
+    public String last() {
+        if (elements.isEmpty()) return "";
+        return elements.get(elements.size() - 1);
+    }
+
+    /**
+     * Returns a new path with the last element replaced by the given element.
+     *
+     * @throws IllegalStateException if this path is empty
+     */
+    public Path withLast(String element) {
+        if (element.isEmpty()) throw new IllegalStateException("Cannot set the last element of an empty path");
+        List<String> newElements = new ArrayList<>(elements);
+        newElements.set(newElements.size() -1, element);
+        return new Path(newElements, delimiter);
+    }
+
+    /** Returns a string representation of this path where the delimiter is prepended */
+    public String getAbsolute() {
+        return delimiter + getRelative();
+    }
+
+    public boolean isRoot() {
+        return elements.isEmpty();
+    }
+
+    public Iterator<String> iterator() { return elements.iterator(); }
+
+    /** Returns an immutable list of the elements of this path in order */
+    public List<String> elements() { return elements; }
+
+    /** Returns this as a string */
+    @Override
+    public String toString() {
+        return getRelative();
+    }
+
+    /**
+     * Creates a path from a string. The string is treated as a relative path, and all redundant '/'-characters are
+     * stripped.
+     *
+     * @param path the relative path that this path should represent
+     * @return a path object that may be used with the application package
+     */
+    public static Path fromString(String path) {
+        return fromString(path, "/");
+    }
+
+    /**
+     * Create a path from a string. The string is treated as a relative path, and all redundant delimiter-characters are
+     * stripped.
+     *
+     * @param path the relative path that this path should represent
+     * @return a path object that may be used with the application package
+     */
+    public static Path fromString(String path, String delimiter) {
+        return new Path(elementsOf(path, delimiter), delimiter);
+    }
+
+    /**
+     * Create an empty root path with '/' delimiter.
+     *
+     * @return an empty root path that can be appended
+     */
+    public static Path createRoot() {
+        return createRoot("/");
+    }
+
+    /**
+     * Create an empty root path with delimiter.
+     *
+     * @return an empty root path that can be appended
+     */
+    public static Path createRoot(String delimiter) {
+        return new Path(delimiter);
+    }
+
+    public File toFile() { return new File(toString()); }
+
+    @Override
+    public int hashCode() {
+        return elements.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other instanceof Path) {
+            return getRelative().equals(((Path) other).getRelative());
+        }
+        return false;
+    }
+
+}
diff --git a/integration/intellij/src/test/applications/rankprofilemodularity/test.sd b/integration/intellij/src/test/applications/rankprofilemodularity/test.sd
index 34414414d6c..34ca174ba5e 100644
--- a/integration/intellij/src/test/applications/rankprofilemodularity/test.sd
+++ b/integration/intellij/src/test/applications/rankprofilemodularity/test.sd
@@ -20,6 +20,11 @@ schema test {
             expression: nativeRank
         }
 
+        function tensorFunction(myTensor) {
+            expression: myTensor
+            # expression: myTensor{key} TODO
+        }
+
     }
 
     rank-profile in_schema2 inherits outside_schema2 {
diff --git a/integration/intellij/src/test/java/ai/vespa/intellij/model/SchemaTest.java b/integration/intellij/src/test/java/ai/vespa/intellij/model/SchemaTest.java
index 5870d71329b..1a33b17d8d3 100644
--- a/integration/intellij/src/test/java/ai/vespa/intellij/model/SchemaTest.java
+++ b/integration/intellij/src/test/java/ai/vespa/intellij/model/SchemaTest.java
@@ -3,7 +3,7 @@ package ai.vespa.intellij.model;
 
 import ai.vespa.intellij.schema.model.RankProfile;
 import ai.vespa.intellij.schema.model.Schema;
-import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
+import ai.vespa.intellij.schema.utils.Path;
 import com.intellij.testFramework.LightProjectDescriptor;
 import com.intellij.testFramework.fixtures.LightJavaCodeInsightFixtureTestCase;
 import org.junit.Test;
@@ -30,11 +30,11 @@ public class SchemaTest extends LightJavaCodeInsightFixtureTestCase {
     @Test
     public void testSimple() {
         super.myFixture.copyDirectoryToProject("src/test/applications/simple", "/");
-        Schema schema = Schema.fromProjectFile(getProject(), "simple.sd");
+        Schema schema = Schema.fromProjectFile(getProject(), Path.fromString("simple.sd"));
         assertNotNull(schema);
-        assertEquals("simple.sd", schema.definition().getName());
-        RankProfile profile = RankProfile.fromProjectFile(getProject(), "simple.sd", "simple-profile");
-        assertEquals("simple-profile", profile.definition().getName());
+        assertEquals("simple", schema.name());
+        RankProfile profile = schema.rankProfile("simple-profile").get();
+        assertEquals("simple-profile", profile.name());
         List<RankProfile> parents = profile.findInherited();
         assertEquals(2, parents.size());
         assertEquals("parent-profile1", parents.get(0).name());
@@ -44,15 +44,29 @@ public class SchemaTest extends LightJavaCodeInsightFixtureTestCase {
     @Test
     public void testSchemaInheritance() {
         super.myFixture.copyDirectoryToProject("src/test/applications/schemainheritance", "/");
-        Schema schema = Schema.fromProjectFile(getProject(), "child.sd");
+        Schema schema = Schema.fromProjectFile(getProject(), Path.fromString("child.sd"));
         assertNotNull(schema);
-        assertEquals("child.sd", schema.definition().getName());
-        RankProfile profile = RankProfile.fromProjectFile(getProject(), "child.sd", "child_profile");
-        assertEquals("child_profile", profile.definition().getName());
+        assertEquals("child", schema.name());
+        RankProfile profile = schema.rankProfile("child_profile").get();
+        assertEquals("child_profile", profile.name());
         List<RankProfile> parents = profile.findInherited();
         assertEquals(2, parents.size());
         assertEquals("other_child_profile", parents.get(0).name());
-        // assertEquals("parent-profile", parents.get(1).name()); TODO
+        assertEquals("parent-profile", parents.get(1).name());
+    }
+
+    @Test
+    public void testRankProfileModularity() {
+        super.myFixture.copyDirectoryToProject("src/test/applications/rankprofilemodularity", "/");
+        Schema schema = Schema.fromProjectFile(getProject(), Path.fromString("test.sd"));
+        assertNotNull(schema);
+        assertEquals("test", schema.name());
+        RankProfile profile = schema.rankProfile("in_schema3").get();
+        assertEquals("in_schema3", profile.name());
+        List<RankProfile> parents = profile.findInherited();
+        assertEquals(2, parents.size());
+        assertEquals("outside_schema1", parents.get(0).name());
+        assertEquals("outside_schema2", parents.get(1).name());
     }
 
 }