Merge pull request #21953 from vespa-engine/jonmv/reapply-more-Path-usage

Use more Path

15 files changed, 87 insertions, 93 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
index 245288c283e..e134b8f53ac 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
@@ -34,11 +34,6 @@ public class DistributableResource {
         this.path = path;
         this.pathType = type;
     }
-    public DistributableResource(String name, ByteBuffer blob) {
-        this.name = name;
-        path = name + ".lz4";
-        pathType = PathType.BLOB;
-    }
 
     //TODO Remove and make path/pathType final
     public void setFileName(String fileName) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java
index 815e4bbf359..6ba17123fb4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java
@@ -6,16 +6,19 @@ import com.yahoo.config.application.api.FileRegistry;
 import java.nio.ByteBuffer;
 import java.util.Objects;
 
+import static java.util.Objects.requireNonNull;
+
 public class RankExpressionBody extends DistributableResource {
 
     private final ByteBuffer blob;
 
     public RankExpressionBody(String name, ByteBuffer blob) {
-        super(name, blob);
-        Objects.requireNonNull(blob, "Blob cannot be null");
-        this.blob = blob;
+        super(name, name + ".lz4", PathType.BLOB);
+        this.blob = requireNonNull(blob, "Blob cannot be null");
     }
+
     public ByteBuffer getBlob() { return blob; }
+
     public void validate() {
         // Remove once pathType is final
         if (getPathType() != PathType.BLOB) {
@@ -26,4 +29,5 @@ public class RankExpressionBody extends DistributableResource {
     void register(FileRegistry fileRegistry) {
         register(fileRegistry, blob);
     }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index e7b7c92ca2f..0f2c1cf9a15 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -5,6 +5,7 @@ import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.path.Path;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.search.query.profile.types.FieldDescription;
 import com.yahoo.search.query.profile.types.QueryProfileType;
@@ -882,10 +883,10 @@ public class RankProfile implements Cloneable {
         if (!expression.startsWith("file:")) return new StringReader(expression);
 
         String fileName = extractFileName(expression);
-        File file = new File(fileName);
-        if (!file.isAbsolute() && file.getPath().contains("/")) // See ticket 4102122
-            throw new IllegalArgumentException("In " + name() + ", " + expName + ", ranking references file '" + file +
-                                               "' in subdirectory, which is not supported.");
+        Path.fromString(fileName); // No ".."
+        if (fileName.contains("/")) // See ticket 4102122
+            throw new IllegalArgumentException("In " + name() + ", " + expName + ", ranking references file '" +
+                                               fileName + "' in a different directory, which is not supported.");
 
         return schema.getRankingExpression(fileName);
     }
diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj
index 8a4798d6f74..6119eb5e2ce 100644
--- a/config-model/src/main/javacc/IntermediateParser.jj
+++ b/config-model/src/main/javacc/IntermediateParser.jj
@@ -1773,7 +1773,7 @@ String fileItem() :
    String path;
 }
 {
-  (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = token.image; } { } (<NL>)*) { return path; }
+  (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = com.yahoo.path.Path.fromString(token.image).getRelative(); } { } (<NL>)*) { return path; }
 }
 String uriItem() :
 {
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index aeffe6e5c39..c4542c36779 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -1907,7 +1907,7 @@ String fileItem() :
    String path;
 }
 {
-  (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = token.image; } { } (<NL>)*) { return path; }
+  (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = com.yahoo.path.Path.fromString(token.image).getRelative(); } { } (<NL>)*) { return path; }
 }
 String uriItem() :
 {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java
index 36a72381156..c686a813c9f 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java
@@ -62,16 +62,27 @@ public class IntermediateParserTestCase {
     @Test
     public void multiple_documents_disallowed() throws Exception {
         String input = joinLines
-            ("schema foo {",
-             "  document foo {",
-             "  }",
-             "  document foo2 {",
-             "  }",
-             "}");
+                ("schema foo {",
+                 "  document foo {",
+                 "  }",
+                 "  document foo2 {",
+                 "  }",
+                 "}");
         var e = assertThrows(IllegalArgumentException.class, () -> parseString(input));
         assertEquals("schema 'foo' error: already has document 'foo' so cannot add document 'foo2'", e.getMessage());
     }
 
+    @Test
+    public void backwards_path_is_disallowed() {
+        assertThrows("'..' is not allowed in path", IllegalArgumentException.class,
+                     () -> parseString("schema foo {\n" +
+                                       "  constant my_constant_tensor {\n" +
+                                       "    file: foo/../bar\n" +
+                                       "    type: tensor<float>(x{},y{})\n" +
+                                       "  }\n" +
+                                       "}\n"));
+    }
+
     void checkFileParses(String fileName) throws Exception {
         System.err.println("TRY parsing: "+fileName);
         var schema = parseFile(fileName);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
index 855b27e0f92..d51713cff77 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
@@ -2,8 +2,8 @@
 package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.config.FileReference;
+import com.yahoo.path.Path;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
@@ -11,8 +11,7 @@ import java.nio.ByteBuffer;
  * @author baldersheim
  */
 public interface AddFileInterface {
-    FileReference addUri(String uri, String relativePath);
-    FileReference addFile(String relativePath) throws IOException;
-    FileReference addFile(File file) throws IOException;
-    FileReference addBlob(ByteBuffer blob, String relativePath);
+    FileReference addUri(String uri, Path path);
+    FileReference addFile(Path path) throws IOException;
+    FileReference addBlob(ByteBuffer blob, Path path);
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
index ad47f2b9e95..b585f7e7e32 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
@@ -3,8 +3,10 @@ package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.config.FileReference;
 import com.yahoo.io.IOUtils;
+import com.yahoo.path.Path;
 import net.jpountz.lz4.LZ4FrameOutputStream;
 
+import java.io.Closeable;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -14,8 +16,6 @@ import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
 import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 
 /**
  * @author baldersheim
@@ -31,56 +31,43 @@ public class ApplicationFileManager implements AddFileInterface {
     }
 
     @Override
-    public FileReference addFile(String relativePath) throws IOException {
-        Path path = Path.of(relativePath).normalize();
-        if (path.isAbsolute())
-            throw new IllegalArgumentException(relativePath + " is not relative");
-        File file = new File(applicationDir, relativePath);
-        Path relative = applicationDir.toPath().relativize(file.toPath()).normalize();
-        if (relative.isAbsolute() || relative.startsWith(".."))
-            throw new IllegalArgumentException(file + " is not a descendant of " + applicationDir);
-
-        return fileDirectory.addFile(file);
+    public FileReference addFile(Path path) throws IOException {
+        File file = new File(applicationDir, path.getRelative());
+        return addFile(file);
     }
 
-    @Override
-    public FileReference addFile(File file) throws IOException {
+    private FileReference addFile(File file) throws IOException {
         return fileDirectory.addFile(file);
     }
 
     @Override
-    public FileReference addUri(String uri, String relativePath) {
-        File file = download(uri, relativePath);
-        try {
-            return addFile(file);
-        } catch (IOException e) {
+    public FileReference addUri(String uri, Path path) {
+        try (TmpDir tmp = new TmpDir()) {
+            return addFile(download(uri, tmp.dir, path));
+        }
+        catch (IOException e) {
             throw new IllegalArgumentException(e);
-        } finally {
-            cleanup(file, relativePath);
         }
     }
 
     @Override
-    public FileReference addBlob(ByteBuffer blob, String relativePath) {
-        File file = writeBlob(blob, relativePath);
-        try {
-            return addFile(file);
-        } catch (IOException e) {
+    public FileReference addBlob(ByteBuffer blob, Path path) {
+        try (TmpDir tmp = new TmpDir()) {
+            return addFile(writeBlob(blob, tmp.dir, path));
+        }
+        catch (IOException e) {
             throw new IllegalArgumentException(e);
-        } finally {
-            cleanup(file, relativePath);
         }
     }
 
-    private File writeBlob(ByteBuffer blob, String relativePath) {
+    private File writeBlob(ByteBuffer blob, File tmpDir, Path path) {
         FileOutputStream fos = null;
         File file = null;
         try {
-            Path path = Files.createTempDirectory("");
-            file = new File(path.toFile(), relativePath);
+            file = new File(tmpDir, path.getRelative());
             Files.createDirectories(file.getParentFile().toPath());
             fos = new FileOutputStream(file);
-            if (relativePath.endsWith(".lz4")) {
+            if (path.last().endsWith(".lz4")) {
                 LZ4FrameOutputStream lz4 = new LZ4FrameOutputStream(fos);
                 lz4.write(blob.array(), blob.arrayOffset(), blob.remaining());
                 lz4.close();
@@ -101,13 +88,12 @@ public class ApplicationFileManager implements AddFileInterface {
         }
     }
 
-    private File download(String uri, String relativePath) {
+    private File download(String uri, File tmpDir, Path path) {
         File file = null;
         FileOutputStream fos = null;
         ReadableByteChannel rbc = null;
         try {
-            Path path = Files.createTempDirectory("");
-            file = new File(path.toFile(), relativePath);
+            file = new File(tmpDir, path.getRelative());
             Files.createDirectories(file.getParentFile().toPath());
             URL website = new URL(uri);
             rbc = Channels.newChannel(website.openStream());
@@ -132,13 +118,10 @@ public class ApplicationFileManager implements AddFileInterface {
         }
     }
 
-    private void cleanup(File file, String relativePath) {
-        Path pathToDelete = file.toPath();
-        // Remove as many components as there are in relative path to find temp path to delete
-        for (int i = 0; i < Paths.get(relativePath).getNameCount(); i++)
-            pathToDelete = pathToDelete.resolveSibling("");
-        IOUtils.recursiveDeleteDir(pathToDelete.toFile());
+    private static class TmpDir implements Closeable {
+        final File dir = Files.createTempDirectory("").toFile();
+        private TmpDir() throws IOException { }
+        @Override public void close() { IOUtils.recursiveDeleteDir(dir); }
     }
 
-
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
index 3eeaa6598d3..c80a6d1d864 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
@@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableMap;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.net.HostName;
+import com.yahoo.path.Path;
 import com.yahoo.text.Utf8;
 import net.jpountz.xxhash.XXHashFactory;
 
@@ -70,10 +71,13 @@ public class FileDBRegistry implements FileRegistry {
 
     @Override
     public synchronized FileReference addFile(String relativePath) {
+        if (relativePath.startsWith("/"))
+            throw new IllegalArgumentException(relativePath + " is not relative");
+
         Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath));
         return cachedReference.orElseGet(() -> {
             try {
-                FileReference newRef = manager.addFile(relativePath);
+                FileReference newRef = manager.addFile(Path.fromString(relativePath));
                 fileReferenceCache.put(relativePath, newRef);
                 return newRef;
             } catch (FileNotFoundException e) {
@@ -94,7 +98,7 @@ public class FileDBRegistry implements FileRegistry {
         String relativePath = uriToRelativeFile(uri);
         Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri));
         return cachedReference.orElseGet(() -> {
-            FileReference newRef = manager.addUri(uri, relativePath);
+            FileReference newRef = manager.addUri(uri, Path.fromString(relativePath));
             fileReferenceCache.put(uri, newRef);
             return newRef;
         });
@@ -106,7 +110,7 @@ public class FileDBRegistry implements FileRegistry {
         synchronized (this) {
             Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(blobName));
             return cachedReference.orElseGet(() -> {
-                FileReference newRef = manager.addBlob(blob, relativePath);
+                FileReference newRef = manager.addBlob(blob, Path.fromString(relativePath));
                 fileReferenceCache.put(blobName, newRef);
                 return newRef;
             });
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
index f4cb7bc1fba..46ae2fd15d5 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
@@ -135,7 +135,7 @@ public class FileDirectory  {
         return new FileReference(Long.toHexString(hash));
     }
 
-    FileReference addFile(File source, FileReference reference) {
+    private FileReference addFile(File source, FileReference reference) {
         ensureRootExist();
         try {
             logfileInfo(source);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileManager.java
index ce0a9866919..6f551e40716 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileManager.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.config.FileReference;
+import com.yahoo.path.Path;
 
 import java.io.File;
 import java.nio.ByteBuffer;
@@ -12,22 +13,17 @@ import java.nio.ByteBuffer;
 public class MockFileManager implements AddFileInterface {
 
     @Override
-    public FileReference addUri(String uri, String relativePath) {
+    public FileReference addUri(String uri, Path path) {
         return null;
     }
 
     @Override
-    public FileReference addFile(String relativePath) {
+    public FileReference addFile(Path path) {
         return null;
     }
 
     @Override
-    public FileReference addFile(File file) {
-        return null;
-    }
-
-    @Override
-    public FileReference addBlob(ByteBuffer blob, String relativePath) {
+    public FileReference addBlob(ByteBuffer blob, Path path) {
         return null;
     }
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java
index cdb01f2013b..6d7074aef3c 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java
@@ -27,7 +27,7 @@ public class FileDBRegistryTestCase {
     private static final String NO_FOO_FILE = "files/no_foo.json";
     private static final String BOO_FILE = "/files/no_foo.json";
     private static final String BAR_FILE = "../files/no_foo.json";
-    private static final String BLOB_NAME = "myblob.name";
+    private static final String BLOB_NAME = "././myblob.name";
     private static final FileReference BLOB_REF = new FileReference("12f292a25163dd9");
     private static final FileReference FOO_REF = new FileReference("b5ce94ca1feae86c");
 
@@ -54,7 +54,7 @@ public class FileDBRegistryTestCase {
             fileRegistry.addFile(BAR_FILE);
             fail();
         } catch (IllegalArgumentException e) {
-            assertEquals("src/test/apps/zkapp/../files/no_foo.json is not a descendant of src/test/apps/zkapp", e.getMessage());
+            assertEquals("'..' is not allowed in path", e.getMessage());
         }
         assertEquals(BLOB_REF, fileRegistry.addBlob(BLOB_NAME, ByteBuffer.wrap(BLOB.getBytes(StandardCharsets.UTF_8))));
         String serializedRegistry = FileDBRegistry.exportRegistry(fileRegistry);
@@ -78,4 +78,5 @@ public class FileDBRegistryTestCase {
     void checkConsistentEntry(FileRegistry.Entry entry, FileRegistry registry) {
         assertEquals(entry.reference, registry.addFile(entry.relativePath));
     }
+    
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java
index 3ac621f7ac4..6c65a2dee81 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java
@@ -17,7 +17,7 @@ public class MockFileDistributionFactory extends FileDistributionFactory {
 
     @Override
     public FileRegistry createFileRegistry(File applicationPackage) {
-        return new MockFileRegistry(applicationPackage, getFileReferencesDir().toPath());
+        return new MockFileRegistry(applicationPackage, getFileReferencesDir());
     }
 
     @Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java
index be2d087478d..9a9c224627a 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java
@@ -3,11 +3,11 @@ package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.path.Path;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -21,18 +21,15 @@ public class MockFileRegistry implements FileRegistry {
     private final List<Entry> entries = new ArrayList<>();
     private final AddFileInterface addFileInterface;
 
-    public MockFileRegistry(File applicationDir, Path rootPath) {
-        FileDirectory fileDirectory = new FileDirectory(rootPath.toFile());
+    public MockFileRegistry(File applicationDir, File rootPath) {
+        FileDirectory fileDirectory = new FileDirectory(rootPath);
         this.addFileInterface = new ApplicationFileManager(applicationDir, fileDirectory);
     }
 
     public FileReference addFile(String relativePath) {
-        if (relativePath.isEmpty())
-            relativePath = "./";
-
+        if (relativePath.isEmpty()) relativePath = "./";
         try {
-            addFileInterface.addFile(relativePath);
-
+            addFileInterface.addFile(Path.fromString(relativePath));
             FileReference fileReference = new FileReference(relativePath);
             entries.add(new Entry(relativePath, fileReference));
             return fileReference;
@@ -50,10 +47,10 @@ public class MockFileRegistry implements FileRegistry {
 
     @Override
     public FileReference addBlob(String name, ByteBuffer blob) {
-        String relativePath = "./" + name;
-        FileReference fileReference = addFileInterface.addBlob(blob, relativePath);
+        name = "./" + name;
+        FileReference fileReference = addFileInterface.addBlob(blob, Path.fromString(name));
 
-        entries.add(new Entry(relativePath, fileReference));
+        entries.add(new Entry(name, fileReference));
         return fileReference;
     }
 
diff --git a/vespajlib/src/test/java/ai/vespa/validation/NameTest.java b/vespajlib/src/test/java/ai/vespa/validation/NameTest.java
index e6645ee4fc3..0b50340870e 100644
--- a/vespajlib/src/test/java/ai/vespa/validation/NameTest.java
+++ b/vespajlib/src/test/java/ai/vespa/validation/NameTest.java
@@ -4,6 +4,9 @@ package ai.vespa.validation;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 /**