diff options
Diffstat (limited to 'configserver')
8 files changed, 53 insertions, 73 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java index 855b27e0f92..d51713cff77 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java @@ -2,8 +2,8 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; +import com.yahoo.path.Path; -import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -11,8 +11,7 @@ import java.nio.ByteBuffer; * @author baldersheim */ public interface AddFileInterface { - FileReference addUri(String uri, String relativePath); - FileReference addFile(String relativePath) throws IOException; - FileReference addFile(File file) throws IOException; - FileReference addBlob(ByteBuffer blob, String relativePath); + FileReference addUri(String uri, Path path); + FileReference addFile(Path path) throws IOException; + FileReference addBlob(ByteBuffer blob, Path path); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java index ad47f2b9e95..b585f7e7e32 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java @@ -3,8 +3,10 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; import net.jpountz.lz4.LZ4FrameOutputStream; +import java.io.Closeable; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -14,8 +16,6 @@ import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; /** * @author baldersheim @@ -31,56 +31,43 @@ public class ApplicationFileManager implements AddFileInterface { } @Override - public FileReference addFile(String relativePath) throws IOException { - Path path = Path.of(relativePath).normalize(); - if (path.isAbsolute()) - throw new IllegalArgumentException(relativePath + " is not relative"); - File file = new File(applicationDir, relativePath); - Path relative = applicationDir.toPath().relativize(file.toPath()).normalize(); - if (relative.isAbsolute() || relative.startsWith("..")) - throw new IllegalArgumentException(file + " is not a descendant of " + applicationDir); - - return fileDirectory.addFile(file); + public FileReference addFile(Path path) throws IOException { + File file = new File(applicationDir, path.getRelative()); + return addFile(file); } - @Override - public FileReference addFile(File file) throws IOException { + private FileReference addFile(File file) throws IOException { return fileDirectory.addFile(file); } @Override - public FileReference addUri(String uri, String relativePath) { - File file = download(uri, relativePath); - try { - return addFile(file); - } catch (IOException e) { + public FileReference addUri(String uri, Path path) { + try (TmpDir tmp = new TmpDir()) { + return addFile(download(uri, tmp.dir, path)); + } + catch (IOException e) { throw new IllegalArgumentException(e); - } finally { - cleanup(file, relativePath); } } @Override - public FileReference addBlob(ByteBuffer blob, String relativePath) { - File file = writeBlob(blob, relativePath); - try { - return addFile(file); - } catch (IOException e) { + public FileReference addBlob(ByteBuffer blob, Path path) { + try (TmpDir tmp = new TmpDir()) { + return addFile(writeBlob(blob, tmp.dir, path)); + } + catch (IOException e) { throw new IllegalArgumentException(e); - } finally { - cleanup(file, relativePath); } } - private File writeBlob(ByteBuffer blob, String relativePath) { + private File writeBlob(ByteBuffer blob, File tmpDir, Path path) { FileOutputStream fos = null; File file = null; try { - Path path = Files.createTempDirectory(""); - file = new File(path.toFile(), relativePath); + file = new File(tmpDir, path.getRelative()); Files.createDirectories(file.getParentFile().toPath()); fos = new FileOutputStream(file); - if (relativePath.endsWith(".lz4")) { + if (path.last().endsWith(".lz4")) { LZ4FrameOutputStream lz4 = new LZ4FrameOutputStream(fos); lz4.write(blob.array(), blob.arrayOffset(), blob.remaining()); lz4.close(); @@ -101,13 +88,12 @@ public class ApplicationFileManager implements AddFileInterface { } } - private File download(String uri, String relativePath) { + private File download(String uri, File tmpDir, Path path) { File file = null; FileOutputStream fos = null; ReadableByteChannel rbc = null; try { - Path path = Files.createTempDirectory(""); - file = new File(path.toFile(), relativePath); + file = new File(tmpDir, path.getRelative()); Files.createDirectories(file.getParentFile().toPath()); URL website = new URL(uri); rbc = Channels.newChannel(website.openStream()); @@ -132,13 +118,10 @@ public class ApplicationFileManager implements AddFileInterface { } } - private void cleanup(File file, String relativePath) { - Path pathToDelete = file.toPath(); - // Remove as many components as there are in relative path to find temp path to delete - for (int i = 0; i < Paths.get(relativePath).getNameCount(); i++) - pathToDelete = pathToDelete.resolveSibling(""); - IOUtils.recursiveDeleteDir(pathToDelete.toFile()); + private static class TmpDir implements Closeable { + final File dir = Files.createTempDirectory("").toFile(); + private TmpDir() throws IOException { } + @Override public void close() { IOUtils.recursiveDeleteDir(dir); } } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index 3eeaa6598d3..c80a6d1d864 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.net.HostName; +import com.yahoo.path.Path; import com.yahoo.text.Utf8; import net.jpountz.xxhash.XXHashFactory; @@ -70,10 +71,13 @@ public class FileDBRegistry implements FileRegistry { @Override public synchronized FileReference addFile(String relativePath) { + if (relativePath.startsWith("/")) + throw new IllegalArgumentException(relativePath + " is not relative"); + Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath)); return cachedReference.orElseGet(() -> { try { - FileReference newRef = manager.addFile(relativePath); + FileReference newRef = manager.addFile(Path.fromString(relativePath)); fileReferenceCache.put(relativePath, newRef); return newRef; } catch (FileNotFoundException e) { @@ -94,7 +98,7 @@ public class FileDBRegistry implements FileRegistry { String relativePath = uriToRelativeFile(uri); Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri)); return cachedReference.orElseGet(() -> { - FileReference newRef = manager.addUri(uri, relativePath); + FileReference newRef = manager.addUri(uri, Path.fromString(relativePath)); fileReferenceCache.put(uri, newRef); return newRef; }); @@ -106,7 +110,7 @@ public class FileDBRegistry implements FileRegistry { synchronized (this) { Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(blobName)); return cachedReference.orElseGet(() -> { - FileReference newRef = manager.addBlob(blob, relativePath); + FileReference newRef = manager.addBlob(blob, Path.fromString(relativePath)); fileReferenceCache.put(blobName, newRef); return newRef; }); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index f4cb7bc1fba..46ae2fd15d5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -135,7 +135,7 @@ public class FileDirectory { return new FileReference(Long.toHexString(hash)); } - FileReference addFile(File source, FileReference reference) { + private FileReference addFile(File source, FileReference reference) { ensureRootExist(); try { logfileInfo(source); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileManager.java index ce0a9866919..6f551e40716 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileManager.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; +import com.yahoo.path.Path; import java.io.File; import java.nio.ByteBuffer; @@ -12,22 +13,17 @@ import java.nio.ByteBuffer; public class MockFileManager implements AddFileInterface { @Override - public FileReference addUri(String uri, String relativePath) { + public FileReference addUri(String uri, Path path) { return null; } @Override - public FileReference addFile(String relativePath) { + public FileReference addFile(Path path) { return null; } @Override - public FileReference addFile(File file) { - return null; - } - - @Override - public FileReference addBlob(ByteBuffer blob, String relativePath) { + public FileReference addBlob(ByteBuffer blob, Path path) { return null; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java index cdb01f2013b..6d7074aef3c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java @@ -27,7 +27,7 @@ public class FileDBRegistryTestCase { private static final String NO_FOO_FILE = "files/no_foo.json"; private static final String BOO_FILE = "/files/no_foo.json"; private static final String BAR_FILE = "../files/no_foo.json"; - private static final String BLOB_NAME = "myblob.name"; + private static final String BLOB_NAME = "././myblob.name"; private static final FileReference BLOB_REF = new FileReference("12f292a25163dd9"); private static final FileReference FOO_REF = new FileReference("b5ce94ca1feae86c"); @@ -54,7 +54,7 @@ public class FileDBRegistryTestCase { fileRegistry.addFile(BAR_FILE); fail(); } catch (IllegalArgumentException e) { - assertEquals("src/test/apps/zkapp/../files/no_foo.json is not a descendant of src/test/apps/zkapp", e.getMessage()); + assertEquals("'..' is not allowed in path", e.getMessage()); } assertEquals(BLOB_REF, fileRegistry.addBlob(BLOB_NAME, ByteBuffer.wrap(BLOB.getBytes(StandardCharsets.UTF_8)))); String serializedRegistry = FileDBRegistry.exportRegistry(fileRegistry); @@ -78,4 +78,5 @@ public class FileDBRegistryTestCase { void checkConsistentEntry(FileRegistry.Entry entry, FileRegistry registry) { assertEquals(entry.reference, registry.addFile(entry.relativePath)); } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java index 3ac621f7ac4..6c65a2dee81 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java @@ -17,7 +17,7 @@ public class MockFileDistributionFactory extends FileDistributionFactory { @Override public FileRegistry createFileRegistry(File applicationPackage) { - return new MockFileRegistry(applicationPackage, getFileReferencesDir().toPath()); + return new MockFileRegistry(applicationPackage, getFileReferencesDir()); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java index be2d087478d..9a9c224627a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java @@ -3,11 +3,11 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.path.Path; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -21,18 +21,15 @@ public class MockFileRegistry implements FileRegistry { private final List<Entry> entries = new ArrayList<>(); private final AddFileInterface addFileInterface; - public MockFileRegistry(File applicationDir, Path rootPath) { - FileDirectory fileDirectory = new FileDirectory(rootPath.toFile()); + public MockFileRegistry(File applicationDir, File rootPath) { + FileDirectory fileDirectory = new FileDirectory(rootPath); this.addFileInterface = new ApplicationFileManager(applicationDir, fileDirectory); } public FileReference addFile(String relativePath) { - if (relativePath.isEmpty()) - relativePath = "./"; - + if (relativePath.isEmpty()) relativePath = "./"; try { - addFileInterface.addFile(relativePath); - + addFileInterface.addFile(Path.fromString(relativePath)); FileReference fileReference = new FileReference(relativePath); entries.add(new Entry(relativePath, fileReference)); return fileReference; @@ -50,10 +47,10 @@ public class MockFileRegistry implements FileRegistry { @Override public FileReference addBlob(String name, ByteBuffer blob) { - String relativePath = "./" + name; - FileReference fileReference = addFileInterface.addBlob(blob, relativePath); + name = "./" + name; + FileReference fileReference = addFileInterface.addBlob(blob, Path.fromString(name)); - entries.add(new Entry(relativePath, fileReference)); + entries.add(new Entry(name, fileReference)); return fileReference; } |