diff options
author | Harald Musum <musum@verizonmedia.com> | 2022-05-25 12:42:07 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-25 12:42:07 +0200 |
commit | 69bb4e0e94cd71aec2b6b20ba146e40394c8a087 (patch) | |
tree | 970a300cf0f34d063c8ac57b0459a3cb77f5a90b /configserver | |
parent | d0824fc978da7240f9a16a3fb4197b21628d9abc (diff) | |
parent | da7c902ab345669dd8f4cf86712c80762202295e (diff) |
Merge pull request #22744 from vespa-engine/hmusum/support-file-distribution-compress-single-file
Support compressing files (not just directories) in file distribution [run-systemtest]
Diffstat (limited to 'configserver')
4 files changed, 44 insertions, 23 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 1edcd3c4d31..14d99cc17af 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -13,14 +13,18 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.filedistribution.FileReferenceCompressor; import com.yahoo.vespa.filedistribution.EmptyFileReferenceData; import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; +import com.yahoo.vespa.filedistribution.FileReferenceCompressor; import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; import com.yahoo.vespa.filedistribution.LazyFileReferenceData; import com.yahoo.vespa.filedistribution.LazyTemporaryStorageFileReferenceData; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.StringFlag; import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; @@ -36,7 +40,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster; -import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; public class FileServer { @@ -48,6 +51,8 @@ public class FileServer { private final FileDirectory root; private final ExecutorService executor; private final FileDownloader downloader; + private final StringFlag compressionAlgorithm; + private final BooleanFlag compressSingleFiles; private enum FileApiErrorCodes { OK(0, "OK"), @@ -80,21 +85,24 @@ public class FileServer { @SuppressWarnings("WeakerAccess") // Created by dependency injection @Inject - public FileServer(ConfigserverConfig configserverConfig) { + public FileServer(ConfigserverConfig configserverConfig, FlagSource flagSource) { this(new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())), - createFileDownloader(getOtherConfigServersInCluster(configserverConfig))); + createFileDownloader(getOtherConfigServersInCluster(configserverConfig)), + flagSource); } // For testing only - public FileServer(File rootDir) { - this(rootDir, createFileDownloader(List.of())); + public FileServer(File rootDir, FlagSource flagSource) { + this(rootDir, createFileDownloader(List.of()), flagSource); } - public FileServer(File rootDir, FileDownloader fileDownloader) { + public FileServer(File rootDir, FileDownloader fileDownloader, FlagSource flagSource) { this.downloader = fileDownloader; this.root = new FileDirectory(rootDir); this.executor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("file-server-")); + this.compressionAlgorithm = Flags.FILE_DISTRIBUTION_COMPRESSION_ALGORITHM.bindTo(flagSource); + this.compressSingleFiles = Flags.FILE_DISTRIBUTION_COMPRESS_SINGLE_FILES.bindTo(flagSource); } boolean hasFile(String fileReference) { @@ -147,17 +155,27 @@ public class FileServer { } private FileReferenceData readFileReferenceData(FileReference reference) throws IOException { + FileReferenceData.Type type = FileReferenceData.from(compressionAlgorithm.value()); File file = root.getFile(reference); - + String fileName = file.getName(); if (file.isDirectory()) { - Path tempFile = Files.createTempFile("filereferencedata", reference.value()); - File compressedFile = new FileReferenceCompressor(compressed).compress(file.getParentFile(), tempFile.toFile()); - return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), compressed, compressedFile); + return createFileReferenceData(file.getParentFile(), reference, type, fileName); + } else if (compressSingleFiles.value()) { + return createFileReferenceData(file, reference, type, fileName); } else { - return new LazyFileReferenceData(reference, file.getName(), FileReferenceData.Type.file, file); + return new LazyFileReferenceData(reference, fileName, FileReferenceData.Type.file, file); } } + LazyTemporaryStorageFileReferenceData createFileReferenceData(File file, + FileReference reference, + FileReferenceData.Type type, + String fileName) throws IOException { + Path tempFile = Files.createTempFile("filereferencedata", reference.value()); + File compressedFile = new FileReferenceCompressor(type).compress(file.getParentFile(), tempFile.toFile()); + return new LazyTemporaryStorageFileReferenceData(reference, fileName, type, compressedFile); + } + public void serveFile(String fileReference, boolean downloadFromOtherSourceIfNotFound, Request request, Receiver receiver) { if (executor instanceof ThreadPoolExecutor) log.log(Level.FINE, () -> "Active threads: " + ((ThreadPoolExecutor) executor).getActiveCount()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index 67c40f94b6a..216a81af0ab 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -10,6 +10,7 @@ import com.yahoo.net.HostName; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -37,7 +38,7 @@ public class FileServerTest { @Before public void setup() throws IOException { File rootDir = new File(temporaryFolder.newFolder("fileserver-root").getAbsolutePath()); - fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir)); + fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir), new InMemoryFlagSource()); } @Test @@ -117,7 +118,7 @@ public class FileServerTest { private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) throws IOException { File fileReferencesDir = temporaryFolder.newFolder(); configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath()); - return new FileServer(new ConfigserverConfig(configBuilder)); + return new FileServer(new ConfigserverConfig(configBuilder), new InMemoryFlagSource()); } private static class FileReceiver implements FileServer.Receiver { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java index 3272689473e..77f67604aad 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.rpc.security.NoopRpcAuthorizer; +import com.yahoo.vespa.flags.InMemoryFlagSource; import java.io.File; import java.time.Duration; @@ -39,7 +40,7 @@ public class MockRpcServer extends RpcServer { Metrics.createTestMetrics(), new HostRegistry(), new ConfigRequestHostLivenessTracker(), - new FileServer(tempDir), + new FileServer(tempDir, new InMemoryFlagSource()), new NoopRpcAuthorizer(), new RpcRequestHandlerProvider()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index 40ed20b7969..73312710c45 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -112,20 +112,21 @@ public class RpcTester implements AutoCloseable { } RpcServer createRpcServer(ConfigserverConfig config) throws IOException { + InMemoryFlagSource flagSource = new InMemoryFlagSource(); RpcServer rpcServer = new RpcServer(config, - new SuperModelRequestHandler(new TestConfigDefinitionRepo(), + new SuperModelRequestHandler(new TestConfigDefinitionRepo(), configserverConfig, new SuperModelManager( config, Zone.defaultZone(), new MemoryGenerationCounter(), - new InMemoryFlagSource())), - Metrics.createTestMetrics(), - hostRegistry, - hostLivenessTracker, - new FileServer(temporaryFolder.newFolder()), - new NoopRpcAuthorizer(), - new RpcRequestHandlerProvider()); + flagSource)), + Metrics.createTestMetrics(), + hostRegistry, + hostLivenessTracker, + new FileServer(temporaryFolder.newFolder(), flagSource), + new NoopRpcAuthorizer(), + new RpcRequestHandlerProvider()); rpcServer.setUpGetConfigHandlers(); return rpcServer; } |